From c20a4daa4c46b8d2334c5ad7eee13d797a60ea85 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 24 Apr 2024 23:16:08 +0100 Subject: [PATCH 01/18] (#1282) Trigger snapshots after robot load --- .../robot_load_then_centre_plan.py | 21 +++++++++++++ tests/conftest.py | 10 ++++++- .../test_wait_for_robot_load_then_centre.py | 30 ++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py index 7fad38041..dccd7096a 100644 --- a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py +++ b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py @@ -2,6 +2,7 @@ import dataclasses import json +from datetime import datetime from typing import cast import bluesky.plan_stubs as bps @@ -26,6 +27,7 @@ from dodal.devices.synchrotron import Synchrotron from dodal.devices.undulator import Undulator from dodal.devices.undulator_dcm import UndulatorDCM +from dodal.devices.webcam import Webcam from dodal.devices.xbpm_feedback import XBPMFeedback from dodal.devices.zebra import Zebra from dodal.devices.zocalo import ZocaloResults @@ -88,6 +90,7 @@ class RobotLoadThenCentreComposite: # RobotLoad fields robot: BartRobot + webcam: Webcam def create_devices(context: BlueskyContext) -> RobotLoadThenCentreComposite: @@ -115,6 +118,18 @@ def wait_for_smargon_not_disabled(smargon: Smargon, timeout=60): ) +def take_robot_snapshots(oav: OAV, webcam: Webcam, directory: str): + time_now = datetime.now() + snapshot_format = f"{time_now.strftime('%H%M%S')}_{{device}}_after_load" + for device in [oav.snapshot, webcam]: + yield from bps.abs_set( + device.filename, snapshot_format.format(device=device.name) + ) + yield from bps.abs_set(device.directory, directory) + yield from bps.trigger(device, group="snapshots") + yield from bps.wait("snapshots") + + def prepare_for_robot_load(composite: RobotLoadThenCentreComposite): yield from bps.abs_set( composite.aperture_scatterguard, @@ -174,8 +189,14 @@ def robot_load(): yield from bps.wait("robot_load") + yield from take_robot_snapshots( + composite.oav, composite.webcam, parameters.experiment_params.snapshot_dir + ) + yield from bps.create(name=CONST.DESCRIPTORS.ROBOT_LOAD) yield from bps.read(composite.robot.barcode) + yield from bps.read(composite.oav.snapshot) + yield from bps.read(composite.webcam) yield from bps.save() yield from wait_for_smargon_not_disabled(composite.smargon) diff --git a/tests/conftest.py b/tests/conftest.py index 098c970e0..7989f8613 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,6 +35,7 @@ from dodal.log import LOGGER as dodal_logger from dodal.log import set_up_all_logging_handlers from ophyd.epics_motor import EpicsMotor +from ophyd.sim import NullStatus from ophyd.status import DeviceStatus, Status from ophyd_async.core import set_sim_value from ophyd_async.core.async_status import AsyncStatus @@ -314,7 +315,9 @@ def synchrotron(): @pytest.fixture def oav(): - return i03.oav(fake_with_ophyd_sim=True) + oav = i03.oav(fake_with_ophyd_sim=True) + oav.snapshot.trigger = MagicMock(return_value=NullStatus()) + return oav @pytest.fixture @@ -405,6 +408,11 @@ def undulator_dcm(): beamline_utils.clear_devices() +@pytest.fixture +def webcam(RE): + return i03.webcam(fake_with_ophyd_sim=True) + + @pytest.fixture def aperture_scatterguard(done_status): AperturePositions.LARGE = SingleAperturePosition( diff --git a/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py b/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py index aa3b42c22..16df2e427 100644 --- a/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py +++ b/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py @@ -5,7 +5,9 @@ from bluesky.utils import Msg from dodal.devices.aperturescatterguard import AperturePositions from dodal.devices.eiger import EigerDetector +from dodal.devices.oav.oav_detector import OAV from dodal.devices.smargon import Smargon, StubPosition +from dodal.devices.webcam import Webcam from numpy import isclose from ophyd.sim import NullStatus, instantiate_fake_device @@ -13,6 +15,7 @@ RobotLoadThenCentreComposite, prepare_for_robot_load, robot_load_then_centre, + take_robot_snapshots, ) from hyperion.external_interaction.callbacks.robot_load.ispyb_callback import ( RobotLoadISPyBCallback, @@ -29,7 +32,7 @@ @pytest.fixture def robot_load_composite( - smargon, dcm, robot, aperture_scatterguard + smargon, dcm, robot, aperture_scatterguard, oav, webcam ) -> RobotLoadThenCentreComposite: composite: RobotLoadThenCentreComposite = MagicMock() composite.smargon = smargon @@ -39,6 +42,8 @@ def robot_load_composite( composite.aperture_scatterguard = aperture_scatterguard composite.smargon.stub_offsets.set = MagicMock(return_value=NullStatus()) composite.aperture_scatterguard.set = MagicMock(return_value=NullStatus()) + composite.oav = oav + composite.webcam = webcam return composite @@ -336,3 +341,26 @@ def test_given_ispyb_callback_attached_when_robot_load_then_centre_plan_called_t start_load.assert_called_once_with("cm31105", 4, "12345", 40, 3) update_barcode.assert_called_once_with(action_id, "BARCODE") end_load.assert_called_once_with(action_id, "success", "") + + +@patch("hyperion.experiment_plans.robot_load_then_centre_plan.datetime") +async def test_when_take_snapshots_called_then_filename_and_directory_set_and_device_triggered( + mock_datetime: MagicMock, oav: OAV, webcam: Webcam +): + TEST_DIRECTORY = "TEST" + + mock_datetime.now.return_value.strftime.return_value = "TIME" + + RE = RunEngine() + oav.snapshot.trigger = MagicMock(side_effect=oav.snapshot.trigger) + webcam.trigger = MagicMock(return_value=NullStatus()) + + RE(take_robot_snapshots(oav, webcam, TEST_DIRECTORY)) + + oav.snapshot.trigger.assert_called_once() + assert oav.snapshot.filename.get() == "TIME_oav_snapshot_after_load" + assert oav.snapshot.directory.get() == TEST_DIRECTORY + + webcam.trigger.assert_called_once() + assert (await webcam.filename.get_value()) == "TIME_webcam_after_load" + assert (await webcam.directory.get_value()) == TEST_DIRECTORY From 094e54eddf327ce4016741d3f9d364fe05e1fcae Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 24 Apr 2024 23:54:21 +0100 Subject: [PATCH 02/18] (#1282) Write robot load snapshots into ispyb --- .../callbacks/robot_load/ispyb_callback.py | 7 ++++++- .../ispyb/exp_eye_store.py | 18 +++++++++++++++--- .../test_wait_for_robot_load_then_centre.py | 13 ++++++++++--- .../test_robot_load_ispyb_callback.py | 18 +++++++++++++++--- .../ispyb/test_expeye_interaction.py | 6 +++++- 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py index 861cf3b92..bd4d36152 100644 --- a/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/robot_load/ispyb_callback.py @@ -62,7 +62,12 @@ def activity_gated_event(self, doc: Event) -> Event | None: self.action_id is not None ), "ISPyB Robot load callback event called unexpectedly" barcode = doc["data"]["robot-barcode"] - self.expeye.update_barcode(self.action_id, barcode) + oav_snapshot = doc["data"]["oav_snapshot_last_saved_path"] + webcam_snapshot = doc["data"]["webcam-last_saved_path"] + # I03 uses oav/webcam snapshots in place of before/after snapshots + self.expeye.update_barcode_and_snapshots( + self.action_id, barcode, oav_snapshot, webcam_snapshot + ) return super().activity_gated_event(doc) diff --git a/src/hyperion/external_interaction/ispyb/exp_eye_store.py b/src/hyperion/external_interaction/ispyb/exp_eye_store.py index 187bbaf50..80a30057b 100644 --- a/src/hyperion/external_interaction/ispyb/exp_eye_store.py +++ b/src/hyperion/external_interaction/ispyb/exp_eye_store.py @@ -81,16 +81,28 @@ def start_load( response = self._send_and_get_response(url, data, post) return response["robotActionId"] - def update_barcode(self, action_id: RobotActionID, barcode: str): - """Update the barcode of an existing robot action. + def update_barcode_and_snapshots( + self, + action_id: RobotActionID, + barcode: str, + snapshot_before_path: str, + snapshot_after_path: str, + ): + """Update the barcode and snapshots of an existing robot action. Args: action_id (RobotActionID): The id of the action to update barcode (str): The barcode to give the action + snapshot_before_path (str): Path to the snapshot before robot load + snapshot_after_path (str): Path to the snapshot after robot load """ url = self.base_url + self.UPDATE_ROBOT_ACTION.format(action_id=action_id) - data = {"sampleBarcode": barcode} + data = { + "sampleBarcode": barcode, + "xtalSnapshotBefore": snapshot_before_path, + "xtalSnapshotAfter": snapshot_after_path, + } self._send_and_get_response(url, data, patch) def end_load(self, action_id: RobotActionID, status: str, reason: str): diff --git a/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py b/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py index 16df2e427..c4969a125 100644 --- a/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py +++ b/tests/unit_tests/experiment_plans/test_wait_for_robot_load_then_centre.py @@ -10,6 +10,7 @@ from dodal.devices.webcam import Webcam from numpy import isclose from ophyd.sim import NullStatus, instantiate_fake_device +from ophyd_async.core import set_sim_value from hyperion.experiment_plans.robot_load_then_centre_plan import ( RobotLoadThenCentreComposite, @@ -310,7 +311,7 @@ def test_when_prepare_for_robot_load_called_then_moves_as_expected( "hyperion.external_interaction.callbacks.robot_load.ispyb_callback.ExpeyeInteraction.end_load" ) @patch( - "hyperion.external_interaction.callbacks.robot_load.ispyb_callback.ExpeyeInteraction.update_barcode" + "hyperion.external_interaction.callbacks.robot_load.ispyb_callback.ExpeyeInteraction.update_barcode_and_snapshots" ) @patch( "hyperion.external_interaction.callbacks.robot_load.ispyb_callback.ExpeyeInteraction.start_load" @@ -325,11 +326,15 @@ def test_when_prepare_for_robot_load_called_then_moves_as_expected( def test_given_ispyb_callback_attached_when_robot_load_then_centre_plan_called_then_ispyb_deposited( mock_centring_plan: MagicMock, start_load: MagicMock, - update_barcode: MagicMock, + update_barcode_and_snapshots: MagicMock, end_load: MagicMock, robot_load_composite: RobotLoadThenCentreComposite, robot_load_then_centre_params: RobotLoadThenCentreInternalParameters, ): + robot_load_composite.oav.snapshot.last_saved_path.put("test_oav_snapshot") # type: ignore + set_sim_value(robot_load_composite.webcam.last_saved_path, "test_webcam_snapshot") + robot_load_composite.webcam.trigger = MagicMock(return_value=NullStatus()) + RE = RunEngine() RE.subscribe(RobotLoadISPyBCallback()) @@ -339,7 +344,9 @@ def test_given_ispyb_callback_attached_when_robot_load_then_centre_plan_called_t RE(robot_load_then_centre(robot_load_composite, robot_load_then_centre_params)) start_load.assert_called_once_with("cm31105", 4, "12345", 40, 3) - update_barcode.assert_called_once_with(action_id, "BARCODE") + update_barcode_and_snapshots.assert_called_once_with( + action_id, "BARCODE", "test_oav_snapshot", "test_webcam_snapshot" + ) end_load.assert_called_once_with(action_id, "success", "") diff --git a/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py index 47e57a4a3..b425f98df 100644 --- a/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/robot_load/test_robot_load_ispyb_callback.py @@ -4,7 +4,10 @@ import bluesky.preprocessors as bpp import pytest from bluesky.run_engine import RunEngine +from dodal.devices.oav.oav_detector import OAV from dodal.devices.robot import BartRobot +from dodal.devices.webcam import Webcam +from ophyd_async.core import set_sim_value from hyperion.external_interaction.callbacks.robot_load.ispyb_callback import ( RobotLoadISPyBCallback, @@ -100,26 +103,35 @@ def test_given_end_called_but_no_start_then_exception_raised(end_load): "hyperion.external_interaction.callbacks.robot_load.ispyb_callback.ExpeyeInteraction.start_load" ) @patch( - "hyperion.external_interaction.callbacks.robot_load.ispyb_callback.ExpeyeInteraction.update_barcode" + "hyperion.external_interaction.callbacks.robot_load.ispyb_callback.ExpeyeInteraction.update_barcode_and_snapshots" ) def test_given_plan_reads_barcode_then_data_put_in_ispyb( - update_barcode: MagicMock, + update_barcode_and_snapshots: MagicMock, start_load: MagicMock, end_load: MagicMock, robot: BartRobot, + oav: OAV, + webcam: Webcam, ): RE = RunEngine() RE.subscribe(RobotLoadISPyBCallback()) start_load.return_value = ACTION_ID + oav.snapshot.last_saved_path.put("test_oav_snapshot") # type: ignore + set_sim_value(webcam.last_saved_path, "test_webcam_snapshot") + @bpp.run_decorator(md=metadata) def my_plan(): yield from bps.create(name=CONST.DESCRIPTORS.ROBOT_LOAD) yield from bps.read(robot.barcode) + yield from bps.read(oav.snapshot) + yield from bps.read(webcam) yield from bps.save() RE(my_plan()) start_load.assert_called_once_with("cm31105", 4, SAMPLE_ID, SAMPLE_PUCK, SAMPLE_PIN) - update_barcode.assert_called_once_with(ACTION_ID, "BARCODE") + update_barcode_and_snapshots.assert_called_once_with( + ACTION_ID, "BARCODE", "test_oav_snapshot", "test_webcam_snapshot" + ) end_load.assert_called_once_with(ACTION_ID, "success", "") diff --git a/tests/unit_tests/external_interaction/ispyb/test_expeye_interaction.py b/tests/unit_tests/external_interaction/ispyb/test_expeye_interaction.py index 2f2fe9e2a..fd56d03c3 100644 --- a/tests/unit_tests/external_interaction/ispyb/test_expeye_interaction.py +++ b/tests/unit_tests/external_interaction/ispyb/test_expeye_interaction.py @@ -125,11 +125,15 @@ def test_when_update_barcode_called_with_success_then_correct_expected_url_poste mock_patch, ): expeye_interactor = ExpeyeInteraction() - expeye_interactor.update_barcode(3, "test") + expeye_interactor.update_barcode_and_snapshots( + 3, "test", "/tmp/before.jpg", "/tmp/after.jpg" + ) mock_patch.assert_called_once() assert mock_patch.call_args.args[0] == "http://blah/core/robot-actions/3" expected_data = { "sampleBarcode": "test", + "xtalSnapshotBefore": "/tmp/before.jpg", + "xtalSnapshotAfter": "/tmp/after.jpg", } assert mock_patch.call_args.kwargs["json"] == expected_data From 88acc8f2a68c25f08a72a6acc3df7d96eac0ecfa Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 25 Apr 2024 11:03:58 +0100 Subject: [PATCH 03/18] (#1282) Fix ispyb system tests for adding snapshots --- .../external_interaction/test_exp_eye_dev.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/system_tests/external_interaction/test_exp_eye_dev.py b/tests/system_tests/external_interaction/test_exp_eye_dev.py index 53337f298..da3fe3aa3 100644 --- a/tests/system_tests/external_interaction/test_exp_eye_dev.py +++ b/tests/system_tests/external_interaction/test_exp_eye_dev.py @@ -10,6 +10,10 @@ @pytest.mark.s03 def test_start_and_end_robot_load(): + """To confirm this test is successful go to + https://ispyb-test.diamond.ac.uk/dc/visit/cm37235-2 and see that data is added + when it's run. + """ os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG SAMPLE_ID = 5289780 @@ -23,7 +27,12 @@ def test_start_and_end_robot_load(): print(f"Created {robot_action_id}") - expeye.update_barcode(robot_action_id, BARCODE) + test_folder = "/dls/i03/data/2024/cm37235-2/xtal_snapshots" + oav_snapshot = test_folder + "/235855_load_after_0.0.png" + webcam_snapshot = test_folder + "/235855_webcam.jpg" + expeye.update_barcode_and_snapshots( + robot_action_id, BARCODE, oav_snapshot, webcam_snapshot + ) sleep(0.5) From cafaf60caa4373818a8b5d370f6ba23621f9708c Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 25 Apr 2024 12:13:17 +0100 Subject: [PATCH 04/18] (#1282) Fix test so that it doesn't use real webcam --- setup.cfg | 2 +- tests/conftest.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index bd7f67afd..7d09abc96 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ install_requires = xarray doct databroker - dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git + dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@7ecd49eb11060a8528ed8b6d31e6c94179bdc2a3 pydantic<2.0 # See https://github.com/DiamondLightSource/hyperion/issues/774 scipy pyzmq<25 # See https://github.com/DiamondLightSource/hyperion/issues/1103 diff --git a/tests/conftest.py b/tests/conftest.py index 7989f8613..8293d8c81 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import sys import threading from functools import partial -from typing import Callable, Generator, Optional, Sequence +from typing import Any, Callable, Generator, Optional, Sequence from unittest.mock import MagicMock, patch import pytest @@ -31,6 +31,7 @@ from dodal.devices.smargon import Smargon from dodal.devices.synchrotron import Synchrotron, SynchrotronMode from dodal.devices.undulator import Undulator +from dodal.devices.webcam import Webcam from dodal.devices.zebra import Zebra from dodal.log import LOGGER as dodal_logger from dodal.log import set_up_all_logging_handlers @@ -409,8 +410,10 @@ def undulator_dcm(): @pytest.fixture -def webcam(RE): - return i03.webcam(fake_with_ophyd_sim=True) +def webcam(RE) -> Generator[Webcam, Any, Any]: + webcam = i03.webcam(fake_with_ophyd_sim=True) + with patch.object(webcam, "_write_image"): + yield webcam @pytest.fixture From 0941349b2e62fae5b9674082659d572720bd319d Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Thu, 4 Apr 2024 12:13:56 +0100 Subject: [PATCH 05/18] (#1283) Populate the microns-per-pixel from OAV params rather than GDA parameters WIP for grid-scan ispyb system test --- .../oav_grid_detection_plan.py | 6 + .../callbacks/ispyb_callback_base.py | 14 +-- tests/conftest.py | 26 +++- .../test_ispyb_dev_connection.py | 117 +++++++++++++++++- .../test_grid_detection_plan.py | 4 + .../callbacks/conftest.py | 4 + 6 files changed, 156 insertions(+), 15 deletions(-) diff --git a/src/hyperion/experiment_plans/oav_grid_detection_plan.py b/src/hyperion/experiment_plans/oav_grid_detection_plan.py index 42e0747a4..f79e7d8e4 100644 --- a/src/hyperion/experiment_plans/oav_grid_detection_plan.py +++ b/src/hyperion/experiment_plans/oav_grid_detection_plan.py @@ -150,6 +150,12 @@ def grid_detection_plan( yield from bps.abs_set(oav.snapshot.box_width, box_size_x_pixels) yield from bps.abs_set(oav.snapshot.num_boxes_x, x_steps) yield from bps.abs_set(oav.snapshot.num_boxes_y, y_steps) + yield from bps.abs_set( + oav.snapshot.microns_per_pixel_x, oav.parameters.micronsPerXPixel + ) + yield from bps.abs_set( + oav.snapshot.microns_per_pixel_y, oav.parameters.micronsPerYPixel + ) snapshot_filename = snapshot_template.format(angle=abs(angle)) diff --git a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py index 2e1e2a7a6..4cc05d9d5 100644 --- a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -159,17 +159,15 @@ def _handle_oav_snapshot_triggered(self, doc) -> Sequence[ScanDataInfo]: data["oav_snapshot_num_boxes_x"] * data["oav_snapshot_num_boxes_y"] ), ) + microns_per_pixel_x = data["oav_snapshot_microns_per_pixel_x"] + microns_per_pixel_y = data["oav_snapshot_microns_per_pixel_y"] data_collection_grid_info = DataCollectionGridInfo( - dx_in_mm=data["oav_snapshot_box_width"] - * self.params.hyperion_params.ispyb_params.microns_per_pixel_x - / 1000, - dy_in_mm=data["oav_snapshot_box_width"] - * self.params.hyperion_params.ispyb_params.microns_per_pixel_y - / 1000, + dx_in_mm=data["oav_snapshot_box_width"] * microns_per_pixel_x / 1000, + dy_in_mm=data["oav_snapshot_box_width"] * microns_per_pixel_y / 1000, steps_x=data["oav_snapshot_num_boxes_x"], steps_y=data["oav_snapshot_num_boxes_y"], - microns_per_pixel_x=self.params.hyperion_params.ispyb_params.microns_per_pixel_x, - microns_per_pixel_y=self.params.hyperion_params.ispyb_params.microns_per_pixel_y, + microns_per_pixel_x=microns_per_pixel_x, + microns_per_pixel_y=microns_per_pixel_y, snapshot_offset_x_pixel=int(data["oav_snapshot_top_left_x"]), snapshot_offset_y_pixel=int(data["oav_snapshot_top_left_y"]), orientation=Orientation.HORIZONTAL, diff --git a/tests/conftest.py b/tests/conftest.py index 8293d8c81..1d99b64d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -262,9 +262,14 @@ def smargon() -> Generator[Smargon, None, None]: smargon.y.user_readback.sim_put(0.0) # type: ignore smargon.z.user_readback.sim_put(0.0) # type: ignore - with patch_motor(smargon.omega), patch_motor(smargon.x), patch_motor( - smargon.y - ), patch_motor(smargon.z), patch_motor(smargon.chi), patch_motor(smargon.phi): + with ( + patch_motor(smargon.omega), + patch_motor(smargon.x), + patch_motor(smargon.y), + patch_motor(smargon.z), + patch_motor(smargon.chi), + patch_motor(smargon.phi), + ): yield smargon @@ -286,6 +291,11 @@ def backlight(): return i03.backlight(fake_with_ophyd_sim=True) +@pytest.fixture +def fast_grid_scan(): + return i03.fast_grid_scan(fake_with_ophyd_sim=True) + + @pytest.fixture def detector_motion(): det = i03.detector_motion(fake_with_ophyd_sim=True) @@ -453,9 +463,13 @@ def aperture_scatterguard(done_status): ) ap_sg.aperture.z.user_setpoint.sim_put(2) # type: ignore ap_sg.aperture.z.motor_done_move.sim_put(1) # type: ignore - with patch_motor(ap_sg.aperture.x), patch_motor(ap_sg.aperture.y), patch_motor( - ap_sg.aperture.z - ), patch_motor(ap_sg.scatterguard.x), patch_motor(ap_sg.scatterguard.y): + with ( + patch_motor(ap_sg.aperture.x), + patch_motor(ap_sg.aperture.y), + patch_motor(ap_sg.aperture.z), + patch_motor(ap_sg.scatterguard.x), + patch_motor(ap_sg.scatterguard.y), + ): ap_sg.set(ap_sg.aperture_positions.SMALL) # type: ignore yield ap_sg diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index 65a315220..f9522d7c3 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -3,7 +3,7 @@ import os from copy import deepcopy from typing import Any, Callable, Literal, Sequence -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from bluesky.run_engine import RunEngine @@ -12,8 +12,14 @@ from dodal.devices.s4_slit_gaps import S4SlitGaps from dodal.devices.synchrotron import Synchrotron, SynchrotronMode from dodal.devices.undulator import Undulator +from ophyd.status import Status from ophyd_async.core import set_sim_value +from hyperion.experiment_plans import oav_grid_detection_plan +from hyperion.experiment_plans.grid_detect_then_xray_centre_plan import ( + GridDetectThenXRayCentreComposite, + grid_detect_then_xray_centre, +) from hyperion.experiment_plans.rotation_scan_plan import ( RotationScanComposite, rotation_scan, @@ -42,6 +48,10 @@ StoreInIspyb, ) from hyperion.parameters.constants import CONST +from hyperion.parameters.external_parameters import from_file +from hyperion.parameters.plan_specific.grid_scan_with_edge_detect_params import ( + GridScanWithEdgeDetectInternalParameters, +) from hyperion.parameters.plan_specific.rotation_scan_internal_params import ( RotationInternalParameters, ) @@ -76,6 +86,97 @@ def dummy_scan_data_info_for_begin(dummy_params): ) +@pytest.fixture +def grid_detect_then_xray_centre_parameters(): + json_dict = from_file( + "tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json" + ) + return GridScanWithEdgeDetectInternalParameters(**json_dict) + + +# noinspection PyUnreachableCode +@pytest.fixture +def grid_detect_then_xray_centre_composite( + fast_grid_scan, + backlight, + smargon, + undulator, + synchrotron, + s4_slit_gaps, + attenuator, + xbpm_feedback, + detector_motion, + zocalo, + aperture_scatterguard, + zebra, + eiger, + robot, + oav, + dcm, + flux, + ophyd_pin_tip_detection, +): + composite = GridDetectThenXRayCentreComposite( + fast_grid_scan=fast_grid_scan, + pin_tip_detection=ophyd_pin_tip_detection, + backlight=backlight, + panda_fast_grid_scan=None, # type: ignore + smargon=smargon, + undulator=undulator, + synchrotron=synchrotron, + s4_slit_gaps=s4_slit_gaps, + attenuator=attenuator, + xbpm_feedback=xbpm_feedback, + detector_motion=detector_motion, + zocalo=zocalo, + aperture_scatterguard=aperture_scatterguard, + zebra=zebra, + eiger=eiger, + panda=None, # type: ignore + robot=robot, + oav=oav, + dcm=dcm, + flux=flux, + ) + eiger.odin.fan.consumers_connected.sim_put(True) + eiger.odin.fan.on.sim_put(True) + eiger.odin.meta.initialised.sim_put(True) + oav.zoom_controller.zrst.set("1.0x") + oav.snapshot.x_size.sim_put(1024) + oav.snapshot.y_size.sim_put(768) + + unpatched_method = oav.parameters.load_microns_per_pixel + + def patch_lmpp(zoom, xsize, ysize): + print("Patch called") + unpatched_method(zoom, 1024, 768) + + def mock_pin_tip_detect(): + yield from [] + return 100, 200 + + with ( + patch.object(eiger.odin.nodes, "get_init_state", return_value=True), + patch.object(eiger, "wait_on_arming_if_started"), + # xsize, ysize will always be wrong since computed as 0 before we get here + # patch up load_microns_per_pixel connect to receive non-zero values + patch.object( + oav.parameters, + "load_microns_per_pixel", + new=MagicMock(side_effect=patch_lmpp), + ), + patch.object( + oav_grid_detection_plan, + "wait_for_tip_to_be_found", + side_effect=mock_pin_tip_detect, + ), + patch.object( + oav.snapshot, "trigger", return_value=Status(success=True, done=True) + ), + ): + yield composite + + def scan_xy_data_info_for_update( data_collection_group_id, dummy_params, scan_data_info_for_begin ): @@ -302,6 +403,20 @@ def generate_scan_data_infos( return scan_data_infos +@pytest.mark.s03 +def test_ispyb_deposition_in_gridscan( + RE: RunEngine, + grid_detect_then_xray_centre_composite: GridDetectThenXRayCentreComposite, + grid_detect_then_xray_centre_parameters: GridScanWithEdgeDetectInternalParameters, +): + RE( + grid_detect_then_xray_centre( + grid_detect_then_xray_centre_composite, + grid_detect_then_xray_centre_parameters, + ) + ) + + @pytest.mark.s03 @patch("bluesky.plan_stubs.wait") def test_ispyb_deposition_in_rotation_plan( diff --git a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py index 6d78c46c2..ae99eb148 100644 --- a/tests/unit_tests/experiment_plans/test_grid_detection_plan.py +++ b/tests/unit_tests/experiment_plans/test_grid_detection_plan.py @@ -250,6 +250,8 @@ def decorated(): "oav_snapshot_num_boxes_x": 8, "oav_snapshot_num_boxes_y": 2, "oav_snapshot_box_width": 16, + "oav_snapshot_microns_per_pixel_x": 1.25, + "oav_snapshot_microns_per_pixel_y": 1.25, "oav_snapshot_last_path_full_overlay": "tmp/test_0_grid_overlay.png", "oav_snapshot_last_path_outer": "tmp/test_0_outer_overlay.png", "oav_snapshot_last_saved_path": "tmp/test_0.png", @@ -263,6 +265,8 @@ def decorated(): "oav_snapshot_num_boxes_x": 8, "oav_snapshot_num_boxes_y": 1, "oav_snapshot_box_width": 16, + "oav_snapshot_microns_per_pixel_x": 1.25, + "oav_snapshot_microns_per_pixel_y": 1.25, "oav_snapshot_last_path_full_overlay": "tmp/test_90_grid_overlay.png", "oav_snapshot_last_path_outer": "tmp/test_90_outer_overlay.png", "oav_snapshot_last_saved_path": "tmp/test_90.png", diff --git a/tests/unit_tests/external_interaction/callbacks/conftest.py b/tests/unit_tests/external_interaction/callbacks/conftest.py index e56c7b185..54fa38cf4 100644 --- a/tests/unit_tests/external_interaction/callbacks/conftest.py +++ b/tests/unit_tests/external_interaction/callbacks/conftest.py @@ -161,6 +161,8 @@ class TestData: "oav_snapshot_top_left_y": 100, "oav_snapshot_num_boxes_x": 40, "oav_snapshot_num_boxes_y": 20, + "oav_snapshot_microns_per_pixel_x": 1.25, + "oav_snapshot_microns_per_pixel_y": 1.25, "oav_snapshot_box_width": 0.1 * 1000 / 1.25, # size in pixels "oav_snapshot_last_path_full_overlay": "test_1_y", "oav_snapshot_last_path_outer": "test_2_y", @@ -178,6 +180,8 @@ class TestData: "oav_snapshot_top_left_y": 0, "oav_snapshot_num_boxes_x": 40, "oav_snapshot_num_boxes_y": 10, + "oav_snapshot_microns_per_pixel_x": 1.25, + "oav_snapshot_microns_per_pixel_y": 1.25, "oav_snapshot_box_width": 0.1 * 1000 / 1.25, # size in pixels "oav_snapshot_last_path_full_overlay": "test_1_z", "oav_snapshot_last_path_outer": "test_2_z", From 8b5f70698d96da07791b3b0df63ad4021d690fba Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 5 Apr 2024 09:59:52 +0100 Subject: [PATCH 06/18] (#1283) Populate the microns-per-pixel from OAV params rather than GDA parameters WIP for grid-scan ispyb system test --- tests/conftest.py | 6 +- .../test_ispyb_dev_connection.py | 200 +++++++++++++++++- ...ispyb_gridscan_system_test_parameters.json | 56 +++++ 3 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json diff --git a/tests/conftest.py b/tests/conftest.py index 1d99b64d8..edfed9664 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,7 @@ from dodal.devices.eiger import EigerDetector from dodal.devices.fast_grid_scan import GridScanCompleteStatus from dodal.devices.flux import Flux +from dodal.devices.oav.oav_detector import OAVConfigParams from dodal.devices.robot import BartRobot from dodal.devices.s4_slit_gaps import S4SlitGaps from dodal.devices.smargon import Smargon @@ -325,9 +326,12 @@ def synchrotron(): @pytest.fixture -def oav(): +def oav(test_config_files): oav = i03.oav(fake_with_ophyd_sim=True) oav.snapshot.trigger = MagicMock(return_value=NullStatus()) + oav.parameters = OAVConfigParams( + test_config_files["zoom_params_file"], test_config_files["display_config"] + ) return oav diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index f9522d7c3..3d030b61b 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -32,6 +32,9 @@ from hyperion.external_interaction.callbacks.rotation.ispyb_callback import ( RotationISPyBCallback, ) +from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( + GridscanISPyBCallback, +) from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( construct_comment_for_gridscan, populate_xy_data_collection_info, @@ -48,7 +51,6 @@ StoreInIspyb, ) from hyperion.parameters.constants import CONST -from hyperion.parameters.external_parameters import from_file from hyperion.parameters.plan_specific.grid_scan_with_edge_detect_params import ( GridScanWithEdgeDetectInternalParameters, ) @@ -58,6 +60,117 @@ from hyperion.utils.utils import convert_angstrom_to_eV from ...conftest import fake_read +from .conftest import raw_params_from_file + +# Map all the case-sensitive column names from their normalised versions +DATA_COLLECTION_COLUMN_MAP = { + s.lower(): s + for s in [ + "dataCollectionId", + "BLSAMPLEID", + "SESSIONID", + "experimenttype", + "dataCollectionNumber", + "startTime", + "endTime", + "runStatus", + "axisStart", + "axisEnd", + "axisRange", + "overlap", + "numberOfImages", + "startImageNumber", + "numberOfPasses", + "exposureTime", + "imageDirectory", + "imagePrefix", + "imageSuffix", + "imageContainerSubPath", + "fileTemplate", + "wavelength", + "resolution", + "detectorDistance", + "xBeam", + "yBeam", + "comments", + "printableForReport", + "CRYSTALCLASS", + "slitGapVertical", + "slitGapHorizontal", + "transmission", + "synchrotronMode", + "xtalSnapshotFullPath1", + "xtalSnapshotFullPath2", + "xtalSnapshotFullPath3", + "xtalSnapshotFullPath4", + "rotationAxis", + "phiStart", + "kappaStart", + "omegaStart", + "chiStart", + "resolutionAtCorner", + "detector2Theta", + "DETECTORMODE", + "undulatorGap1", + "undulatorGap2", + "undulatorGap3", + "beamSizeAtSampleX", + "beamSizeAtSampleY", + "centeringMethod", + "averageTemperature", + "ACTUALSAMPLEBARCODE", + "ACTUALSAMPLESLOTINCONTAINER", + "ACTUALCONTAINERBARCODE", + "ACTUALCONTAINERSLOTINSC", + "actualCenteringPosition", + "beamShape", + "dataCollectionGroupId", + "POSITIONID", + "detectorId", + "FOCALSPOTSIZEATSAMPLEX", + "POLARISATION", + "FOCALSPOTSIZEATSAMPLEY", + "APERTUREID", + "screeningOrigId", + "flux", + "strategySubWedgeOrigId", + "blSubSampleId", + "processedDataFile", + "datFullPath", + "magnification", + "totalAbsorbedDose", + "binning", + "particleDiameter", + "boxSize", + "minResolution", + "minDefocus", + "maxDefocus", + "defocusStepSize", + "amountAstigmatism", + "extractSize", + "bgRadius", + "voltage", + "objAperture", + "c1aperture", + "c2aperture", + "c3aperture", + "c1lens", + "c2lens", + "c3lens", + "startPositionId", + "endPositionId", + "flux", + "bestWilsonPlotPath", + "totalExposedDose", + "nominalMagnification", + "nominalDefocus", + "imageSizeX", + "imageSizeY", + "pixelSizeOnImage", + "phasePlate", + "dataCollectionPlanId", + ] +} @pytest.fixture @@ -88,8 +201,8 @@ def dummy_scan_data_info_for_begin(dummy_params): @pytest.fixture def grid_detect_then_xray_centre_parameters(): - json_dict = from_file( - "tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json" + json_dict = raw_params_from_file( + "tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json" ) return GridScanWithEdgeDetectInternalParameters(**json_dict) @@ -144,17 +257,31 @@ def grid_detect_then_xray_centre_composite( oav.zoom_controller.zrst.set("1.0x") oav.snapshot.x_size.sim_put(1024) oav.snapshot.y_size.sim_put(768) - unpatched_method = oav.parameters.load_microns_per_pixel + eiger.stale_params.sim_put(0) + eiger.odin.meta.ready.sim_put(1) + eiger.odin.fan.ready.sim_put(1) def patch_lmpp(zoom, xsize, ysize): print("Patch called") unpatched_method(zoom, 1024, 768) - def mock_pin_tip_detect(): + def mock_pin_tip_detect(_): yield from [] return 100, 200 + def mock_set_file_name(val, timeout): + eiger.odin.meta.file_name.sim_put(val) # type: ignore + eiger.odin.file_writer.id.sim_put(val) # type: ignore + return Status(success=True, done=True) + + unpatched_complete = fast_grid_scan.complete + + def mock_complete_status(): + status = unpatched_complete() + status.set_finished() + return status + with ( patch.object(eiger.odin.nodes, "get_init_state", return_value=True), patch.object(eiger, "wait_on_arming_if_started"), @@ -173,6 +300,16 @@ def mock_pin_tip_detect(): patch.object( oav.snapshot, "trigger", return_value=Status(success=True, done=True) ), + patch.object( + eiger.odin.file_writer.file_name, + "set", + side_effect=mock_set_file_name, + ), + patch.object( + fast_grid_scan, "kickoff", return_value=Status(success=True, done=True) + ), + patch.object(fast_grid_scan, "complete", side_effect=mock_complete_status), + patch.object(zocalo, "trigger", return_value=Status(success=True, done=True)), ): yield composite @@ -408,7 +545,13 @@ def test_ispyb_deposition_in_gridscan( RE: RunEngine, grid_detect_then_xray_centre_composite: GridDetectThenXRayCentreComposite, grid_detect_then_xray_centre_parameters: GridScanWithEdgeDetectInternalParameters, + fetch_datacollection_attribute: Callable[..., Any], ): + os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG + grid_detect_then_xray_centre_composite.s4_slit_gaps.xgap.user_readback.sim_put(0.1) + grid_detect_then_xray_centre_composite.s4_slit_gaps.ygap.user_readback.sim_put(0.1) + ispyb_callback = GridscanISPyBCallback() + RE.subscribe(ispyb_callback) RE( grid_detect_then_xray_centre( grid_detect_then_xray_centre_composite, @@ -416,6 +559,53 @@ def test_ispyb_deposition_in_gridscan( ) ) + ispyb_ids = ispyb_callback.ispyb_ids + dcid = ispyb_ids.data_collection_ids[0] + expected_values = { + # "visitid": TEST_SESSION_ID, + # "parentid": TEST_DATA_COLLECTION_GROUP_ID, + # "sampleid": TEST_SAMPLE_ID, + "detectorid": 78, + "axisstart": 0.0, + "axisrange": 0, + "axisend": 0, + "focalspotsizeatsamplex": 1.0, + "focalspotsizeatsampley": 1.0, + "slitgapvertical": 0.1, + "slitgaphorizontal": 0.1, + "beamsizeatsamplex": 1, + "beamsizeatsampley": 1, + "transmission": 49.118, + "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " + "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " + "bottom right (px): [3250,1700].", + "datacollectionnumber": 0, + "detectordistance": 100.0, + "exptime": 0.1, + "imgdir": "/tmp/", + "imgprefix": "file_name", + "imgsuffix": "h5", + "npasses": 1, + "overlap": 0, + "omegastart": 0, + "startimagenumber": 1, + "resolution": 1.0, # deferred + "wavelength": 123.98419840550369, + "xbeam": 150.0, + "ybeam": 160.0, + "xtalsnapshot1": "test1y", + "xtalsnapshot2": "test2y", + "xtalsnapshot3": "test3y", + "synchrotronmode": None, + "undulatorgap1": 1.0, + # "starttime": EXPECTED_START_TIME, + "filetemplate": "file_name_0_master.h5", + "nimages": 40 * 20, + } + for k, v in expected_values.items(): + actual = fetch_datacollection_attribute(dcid, DATA_COLLECTION_COLUMN_MAP[k]) + assert actual == v, f"expected {k} {actual} == {v}" + @pytest.mark.s03 @patch("bluesky.plan_stubs.wait") diff --git a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json new file mode 100644 index 000000000..416cf6c67 --- /dev/null +++ b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json @@ -0,0 +1,56 @@ +{ + "params_version": "4.0.5", + "hyperion_params": { + "beamline": "BL03S", + "insertion_prefix": "SR03S", + "detector": "EIGER2_X_16M", + "zocalo_environment": "dev_artemis", + "experiment_type": "grid_detect_then_xray_centre", + "detector_params": { + "expected_energy_ev": 100, + "directory": "/tmp", + "prefix": "file_name", + "run_number": 0, + "use_roi_mode": false, + "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt" + }, + "ispyb_params": { + "visit_path": "/tmp/cm31105-4/", + "position": [ + 10.0, + 20.0, + 30.0 + ], + "xtal_snapshots_omega_start": [ + "test_1_y", + "test_2_y", + "test_3_y" + ], + "xtal_snapshots_omega_end": [ + "test_1_z", + "test_2_z", + "test_3_z" + ], + "transmission_fraction": 1.0, + "beam_size_x": 1.0, + "beam_size_y": 1.0, + "focal_spot_size_x": 1.0, + "focal_spot_size_y": 1.0, + "comment": "test", + "resolution": 1.0 + } + }, + "experiment_params": { +// "x_steps": 40, +// "y_steps": 20, +// "z_steps": 10, + "x_step_size": 0.1, + "y_step_size": 0.1, + "z_step_size": 0.1, + "snapshot_dir": "/tmp", + "exposure_time": 0.1, + "detector_distance": 100.0, + "omega_start": 0.0, + "grid_width_microns": 151 + } +} \ No newline at end of file From 1b252ca7d472e944ee5097fae2b34209993293d1 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 8 Apr 2024 10:20:50 +0100 Subject: [PATCH 07/18] (#1283) Populate the microns-per-pixel from OAV params rather than GDA parameters * Remove support for 2D gridscans from ispyb callbacks * Remove references to microns-per-pixel from ispyb params * Ispyb system test for grid scan and grid-detect --- .../callbacks/ispyb_callback_base.py | 2 +- .../callbacks/xray_centre/ispyb_callback.py | 67 ++++---- .../callbacks/xray_centre/ispyb_mapping.py | 12 +- tests/conftest.py | 6 +- .../test_ispyb_dev_connection.py | 145 ++++++++++++++---- ...ispyb_gridscan_system_test_parameters.json | 15 +- .../xray_centre/test_ispyb_callback.py | 113 -------------- .../xray_centre/test_ispyb_mapping.py | 14 +- 8 files changed, 170 insertions(+), 204 deletions(-) diff --git a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py index 4cc05d9d5..630595098 100644 --- a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -174,7 +174,7 @@ def _handle_oav_snapshot_triggered(self, doc) -> Sequence[ScanDataInfo]: snaked=True, ) data_collection_info.comments = construct_comment_for_gridscan( - self.params.hyperion_params.ispyb_params, data_collection_grid_info + data_collection_grid_info ) if len(self.ispyb_ids.data_collection_ids) > self._oav_snapshot_event_idx: data_collection_id = self.ispyb_ids.data_collection_ids[ diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py index 98b8daa0c..7d4cab607 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py @@ -83,9 +83,6 @@ def __init__( self._start_of_fgs_uid: str | None = None self._processing_start_time: float | None = None - def is_3d_gridscan(self): - return self.params.experiment_params.is_3d_grid_scan - def activity_gated_start(self, doc: RunStart): if doc.get("subplan_name") == CONST.PLAN.DO_FGS: self._start_of_fgs_uid = doc.get("uid") @@ -97,14 +94,7 @@ def activity_gated_start(self, doc: RunStart): ) json_params = doc.get("hyperion_internal_parameters") self.params = GridscanInternalParameters.from_json(json_params) - self.ispyb = StoreInIspyb( - self.ispyb_config, - ( - ExperimentType.GRIDSCAN_3D - if self.is_3d_gridscan() - else ExperimentType.GRIDSCAN_2D - ), - ) + self.ispyb = StoreInIspyb(self.ispyb_config, ExperimentType.GRIDSCAN_3D) data_collection_group_info = populate_data_collection_group( self.ispyb.experiment_type, self.params.hyperion_params.detector_params, @@ -122,22 +112,20 @@ def activity_gated_start(self, doc: RunStart): self.params.hyperion_params.detector_params, self.params.hyperion_params.ispyb_params, ), - ) - ] - if self.is_3d_gridscan(): - scan_data_infos.append( - ScanDataInfo( - data_collection_info=populate_remaining_data_collection_info( - None, - None, - populate_xz_data_collection_info( - self.params.hyperion_params.detector_params - ), - self.params.hyperion_params.detector_params, - self.params.hyperion_params.ispyb_params, - ) + ), + ScanDataInfo( + data_collection_info=populate_remaining_data_collection_info( + None, + None, + populate_xz_data_collection_info( + + self.params.hyperion_params.detector_params + ), + self.params.hyperion_params.detector_params, + self.params.hyperion_params.ispyb_params, ) - ) + ), + ] self.ispyb_ids = self.ispyb.begin_deposition( data_collection_group_info, scan_data_infos @@ -200,20 +188,19 @@ def populate_info_for_update( ) scan_data_infos = [xy_scan_data_info] - if self.is_3d_gridscan(): - data_collection_id = ( - self.ispyb_ids.data_collection_ids[1] - if len(self.ispyb_ids.data_collection_ids) > 1 - else None - ) - xz_scan_data_info = ScanDataInfo( - data_collection_info=event_sourced_data_collection_info, - data_collection_position_info=populate_data_collection_position_info( - params.hyperion_params.ispyb_params - ), - data_collection_id=data_collection_id, - ) - scan_data_infos.append(xz_scan_data_info) + data_collection_id = ( + self.ispyb_ids.data_collection_ids[1] + if len(self.ispyb_ids.data_collection_ids) > 1 + else None + ) + xz_scan_data_info = ScanDataInfo( + data_collection_info=event_sourced_data_collection_info, + data_collection_position_info=populate_data_collection_position_info( + params.hyperion_params.ispyb_params + ), + data_collection_id=data_collection_id, + ) + scan_data_infos.append(xz_scan_data_info) return scan_data_infos def activity_gated_stop(self, doc: RunStop) -> RunStop: diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py index 5700d20fc..3a5c53bde 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py @@ -35,12 +35,8 @@ def populate_xy_data_collection_info(detector_params): return info -def construct_comment_for_gridscan( - ispyb_params, grid_info: DataCollectionGridInfo -) -> str: - assert ( - ispyb_params is not None and grid_info is not None - ), "StoreGridScanInIspyb failed to get parameters" +def construct_comment_for_gridscan(grid_info: DataCollectionGridInfo) -> str: + assert grid_info is not None, "StoreGridScanInIspyb failed to get parameters" bottom_right = oav_utils.bottom_right_from_top_left( numpy.array( @@ -50,8 +46,8 @@ def construct_comment_for_gridscan( grid_info.steps_y, grid_info.dx_in_mm, grid_info.dy_in_mm, - ispyb_params.microns_per_pixel_x, - ispyb_params.microns_per_pixel_y, + grid_info.microns_per_pixel_x, + grid_info.microns_per_pixel_y, ) return ( "Hyperion: Xray centring - Diffraction grid scan of " diff --git a/tests/conftest.py b/tests/conftest.py index edfed9664..0151df580 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -327,11 +327,11 @@ def synchrotron(): @pytest.fixture def oav(test_config_files): - oav = i03.oav(fake_with_ophyd_sim=True) - oav.snapshot.trigger = MagicMock(return_value=NullStatus()) - oav.parameters = OAVConfigParams( + parameters = OAVConfigParams( test_config_files["zoom_params_file"], test_config_files["display_config"] ) + oav = i03.oav(fake_with_ophyd_sim=True, params=parameters) + oav.snapshot.trigger = MagicMock(return_value=NullStatus()) return oav diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index 3d030b61b..821e6724e 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re from copy import deepcopy from typing import Any, Callable, Literal, Sequence from unittest.mock import MagicMock, patch @@ -255,20 +256,47 @@ def grid_detect_then_xray_centre_composite( eiger.odin.fan.on.sim_put(True) eiger.odin.meta.initialised.sim_put(True) oav.zoom_controller.zrst.set("1.0x") + oav.cam.array_size.array_size_x.sim_put(1024) + oav.cam.array_size.array_size_y.sim_put(768) oav.snapshot.x_size.sim_put(1024) oav.snapshot.y_size.sim_put(768) + oav.snapshot.top_left_x.set(50) + oav.snapshot.top_left_y.set(100) + oav.snapshot.box_width.set(0.1 * 1000 / 1.25) # size in pixels + oav.snapshot.last_path_full_overlay.set("test_1_y") + oav.snapshot.last_path_outer.set("test_2_y") + oav.snapshot.last_saved_path.set("test_3_y") + undulator.current_gap.sim_put(1.11) + unpatched_method = oav.parameters.load_microns_per_pixel eiger.stale_params.sim_put(0) eiger.odin.meta.ready.sim_put(1) eiger.odin.fan.ready.sim_put(1) def patch_lmpp(zoom, xsize, ysize): - print("Patch called") unpatched_method(zoom, 1024, 768) def mock_pin_tip_detect(_): + tip_x_px = 100 + tip_y_px = 200 + microns_per_pixel = 2.87 # from zoom levels .xml + grid_width_px = int(400 / microns_per_pixel) + target_grid_height_px = 70 + top_edge_data = ([0] * tip_x_px) + ( + [(tip_y_px - target_grid_height_px // 2)] * grid_width_px + ) + bottom_edge_data = [0] * tip_x_px + [ + (tip_y_px + target_grid_height_px // 2) + ] * grid_width_px + ophyd_pin_tip_detection.triggered_top_edge._backend._set_value( + numpy.array(top_edge_data, dtype=numpy.uint32) + ) + + ophyd_pin_tip_detection.triggered_bottom_edge._backend._set_value( + numpy.array(bottom_edge_data, dtype=numpy.uint32) + ) yield from [] - return 100, 200 + return tip_x_px, tip_y_px def mock_set_file_name(val, timeout): eiger.odin.meta.file_name.sim_put(val) # type: ignore @@ -336,7 +364,6 @@ def scan_xy_data_info_for_update( ) scan_data_info_for_update.data_collection_info.comments = ( construct_comment_for_gridscan( - dummy_params.hyperion_params.ispyb_params, scan_data_info_for_update.data_collection_grid_info, ) ) @@ -371,9 +398,7 @@ def scan_data_infos_for_update_3d( snaked=True, ) xz_data_collection_info = populate_remaining_data_collection_info( - construct_comment_for_gridscan( - dummy_params.hyperion_params.ispyb_params, data_collection_grid_info - ), + construct_comment_for_gridscan(data_collection_grid_info), ispyb_ids.data_collection_group_id, xz_data_collection_info, dummy_params.hyperion_params.detector_params, @@ -560,11 +585,7 @@ def test_ispyb_deposition_in_gridscan( ) ispyb_ids = ispyb_callback.ispyb_ids - dcid = ispyb_ids.data_collection_ids[0] expected_values = { - # "visitid": TEST_SESSION_ID, - # "parentid": TEST_DATA_COLLECTION_GROUP_ID, - # "sampleid": TEST_SAMPLE_ID, "detectorid": 78, "axisstart": 0.0, "axisrange": 0, @@ -576,35 +597,103 @@ def test_ispyb_deposition_in_gridscan( "beamsizeatsamplex": 1, "beamsizeatsampley": 1, "transmission": 49.118, - "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " - "bottom right (px): [3250,1700].", "datacollectionnumber": 0, "detectordistance": 100.0, - "exptime": 0.1, - "imgdir": "/tmp/", - "imgprefix": "file_name", - "imgsuffix": "h5", - "npasses": 1, + "exposuretime": 0.12, + "imagedirectory": "/tmp/", + "imageprefix": "file_name", + "imagesuffix": "h5", + "numberofpasses": 1, "overlap": 0, "omegastart": 0, "startimagenumber": 1, - "resolution": 1.0, # deferred - "wavelength": 123.98419840550369, + "resolution": 1.0, + "wavelength": 0.976254, "xbeam": 150.0, "ybeam": 160.0, - "xtalsnapshot1": "test1y", - "xtalsnapshot2": "test2y", - "xtalsnapshot3": "test3y", - "synchrotronmode": None, - "undulatorgap1": 1.0, - # "starttime": EXPECTED_START_TIME, + "xtalsnapshotfullpath1": "test_1_y", + "xtalsnapshotfullpath2": "test_2_y", + "xtalsnapshotfullpath3": "test_3_y", + "synchrotronmode": "User", + "undulatorgap1": 1.11, "filetemplate": "file_name_0_master.h5", - "nimages": 40 * 20, + "numberofimages": 20 * 12, + } + compare_comment( + fetch_datacollection_attribute, + ispyb_ids.data_collection_ids[0], + "Hyperion: Xray centring - Diffraction grid scan of 20 by 12 " + "images in 20.0 um by 20.0 um steps. Top left (px): [100,161], " + "bottom right (px): [239,245].", + ) + compare_actual_and_expected( + ispyb_ids.data_collection_ids[0], + expected_values, + fetch_datacollection_attribute, + ) + expected_values = { + "detectorid": 78, + "axisstart": 90.0, + "axisrange": 0, + "axisend": 90, + "focalspotsizeatsamplex": 1.0, + "focalspotsizeatsampley": 1.0, + "slitgapvertical": 0.1, + "slitgaphorizontal": 0.1, + "beamsizeatsamplex": 1, + "beamsizeatsampley": 1, + "transmission": 49.118, + "datacollectionnumber": 1, + "detectordistance": 100.0, + "exposuretime": 0.12, + "imagedirectory": "/tmp/", + "imageprefix": "file_name", + "imagesuffix": "h5", + "numberofpasses": 1, + "overlap": 0, + "omegastart": 90, + "startimagenumber": 1, + "resolution": 1.0, + "wavelength": 0.976254, + "xbeam": 150.0, + "ybeam": 160.0, + "xtalsnapshotfullpath1": "test_1_y", + "xtalsnapshotfullpath2": "test_2_y", + "xtalsnapshotfullpath3": "test_3_y", + "synchrotronmode": "User", + "undulatorgap1": 1.11, + "filetemplate": "file_name_1_master.h5", + "numberofimages": 20 * 11, } + compare_actual_and_expected( + ispyb_ids.data_collection_ids[1], + expected_values, + fetch_datacollection_attribute, + ) + compare_comment( + fetch_datacollection_attribute, + ispyb_ids.data_collection_ids[1], + "Hyperion: Xray centring - Diffraction grid scan of 20 by 11 " + "images in 20.0 um by 20.0 um steps. Top left (px): [100,165], " + "bottom right (px): [239,241].", + ) + + +def compare_comment( + fetch_datacollection_attribute, data_collection_id, expected_comment +): + actual_comment = fetch_datacollection_attribute( + data_collection_id, DATA_COLLECTION_COLUMN_MAP["comments"] + ) + match = re.search(" Zocalo processing took", actual_comment) + truncated_comment = actual_comment[: match.start()] if match else actual_comment + assert truncated_comment == expected_comment + + +def compare_actual_and_expected(dcid, expected_values, fetch_datacollection_attribute): for k, v in expected_values.items(): actual = fetch_datacollection_attribute(dcid, DATA_COLLECTION_COLUMN_MAP[k]) - assert actual == v, f"expected {k} {actual} == {v}" + assert actual == v, f"expected {k} {v} == {actual}" @pytest.mark.s03 diff --git a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json index 416cf6c67..6e352ada2 100644 --- a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json +++ b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json @@ -37,20 +37,19 @@ "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", - "resolution": 1.0 + "resolution": 1.0, + "microns_per_pixel_x": -1, + "microns_per_pixel_y": -1 } }, "experiment_params": { -// "x_steps": 40, -// "y_steps": 20, -// "z_steps": 10, - "x_step_size": 0.1, - "y_step_size": 0.1, - "z_step_size": 0.1, + "z_steps": 10, "snapshot_dir": "/tmp", "exposure_time": 0.1, "detector_distance": 100.0, "omega_start": 0.0, - "grid_width_microns": 151 + "grid_width_microns": 400, + "exposure_time": 0.12, + "requested_energy_kev": 12.7 } } \ No newline at end of file diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py index 375592822..e37e057b8 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py @@ -94,119 +94,6 @@ new=MagicMock(return_value=EXPECTED_START_TIME), ) class TestXrayCentreISPyBCallback: - def test_activity_gated_start_2d(self, mock_ispyb_conn): - callback = GridscanISPyBCallback() - callback.activity_gated_start( - TestData.test_gridscan2d_start_document # pyright: ignore - ) - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "parentid": TEST_SESSION_ID, - "experimenttype": "mesh", - "sampleid": TEST_SAMPLE_ID, - }, - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - EXPECTED_DATA_COLLECTION_2D, - ) - mx_acq.upsert_data_collection.update_dc_position.assert_not_called() - mx_acq.upsert_data_collection.upsert_dc_grid.assert_not_called() - - def test_hardware_read_event_2d(self, mock_ispyb_conn): - callback = GridscanISPyBCallback() - callback.activity_gated_start( - TestData.test_gridscan2d_start_document # pyright: ignore - ) - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - mx_acq.upsert_data_collection_group.reset_mock() - mx_acq.upsert_data_collection.reset_mock() - callback.activity_gated_descriptor( - TestData.test_descriptor_document_pre_data_collection - ) - callback.activity_gated_event(TestData.test_event_document_pre_data_collection) - - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "mesh", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": "BARCODE", # deferred - }, - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - { - "parentid": TEST_DATA_COLLECTION_GROUP_ID, - "id": TEST_DATA_COLLECTION_IDS[0], - "slitgaphorizontal": 0.1234, - "slitgapvertical": 0.2345, - "synchrotronmode": "User", - "undulatorgap1": 1.234, - }, - ) - assert_upsert_call_with( - mx_acq.update_dc_position.mock_calls[0], - mx_acq.get_dc_position_params(), - { - "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": 0, - "pos_y": 0, - "pos_z": 0, - }, - ) - mx_acq.upsert_dc_grid.assert_not_called() - - def test_flux_read_event_2d(self, mock_ispyb_conn): - callback = GridscanISPyBCallback() - callback.activity_gated_start( - TestData.test_gridscan2d_start_document # pyright: ignore - ) - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - callback.activity_gated_descriptor( - TestData.test_descriptor_document_pre_data_collection - ) - callback.activity_gated_event(TestData.test_event_document_pre_data_collection) - mx_acq.upsert_data_collection_group.reset_mock() - mx_acq.upsert_data_collection.reset_mock() - callback.activity_gated_descriptor( - TestData.test_descriptor_document_during_data_collection - ) - callback.activity_gated_event( - TestData.test_event_document_during_data_collection - ) - - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - { - "parentid": TEST_DATA_COLLECTION_GROUP_ID, - "id": TEST_DATA_COLLECTION_IDS[0], - "wavelength": 1.1164718451643736, - "transmission": 100, - "flux": 10, - }, - ) - assert_upsert_call_with( - mx_acq.update_dc_position.mock_calls[0], - mx_acq.get_dc_position_params(), - { - "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": 0, - "pos_y": 0, - "pos_z": 0, - }, - ) - mx_acq.upsert_dc_grid.assert_not_called() - def test_activity_gated_start_3d(self, mock_ispyb_conn): callback = GridscanISPyBCallback() callback.activity_gated_start( diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py index 8867c2a5d..295a32078 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py @@ -33,9 +33,17 @@ def test_ispyb_deposition_rounds_position_to_int( dummy_params, ): assert construct_comment_for_gridscan( - dummy_params.hyperion_params.ispyb_params, DataCollectionGridInfo( - 0.1, 0.1, 40, 20, 1.25, 1.25, 0.01, 100, Orientation.HORIZONTAL, True # type: ignore + 0.1, + 0.1, + 40, + 20, + 1.25, + 1.25, + 0.01, + 100, + Orientation.HORIZONTAL, + True, # type: ignore ), ) == ( "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images " @@ -70,7 +78,7 @@ def test_ispyb_deposition_rounds_box_size_int( ) bottom_right_from_top_left.return_value = [0, 0] - assert construct_comment_for_gridscan(MagicMock(), data_collection_grid_info) == ( + assert construct_comment_for_gridscan(data_collection_grid_info) == ( "Hyperion: Xray centring - Diffraction grid scan of 0 by 0 images in " f"{rounded} um by {rounded} um steps. Top left (px): [0,0], bottom right (px): [0,0]." ) From a1ce4c76347997d786a8f576a84c91131f931c9c Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 8 Apr 2024 14:18:31 +0100 Subject: [PATCH 08/18] (#1283) Fix system test breakage after rebase due to transmission_fraction --- .../ispyb_gridscan_system_test_parameters.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json index 6e352ada2..e6b4930b5 100644 --- a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json +++ b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json @@ -45,7 +45,7 @@ "experiment_params": { "z_steps": 10, "snapshot_dir": "/tmp", - "exposure_time": 0.1, + "transmission_fraction": 1.0, "detector_distance": 100.0, "omega_start": 0.0, "grid_width_microns": 400, From 00f5951e41b7578df3b801beec8c735a6cbc4e9b Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Tue, 9 Apr 2024 09:54:30 +0100 Subject: [PATCH 09/18] (#1283) Add system test for grid and position info Remove microns-per-pixel from ispyb_params Fix minor issue where grid-ids were not being returned in the IspybIds --- .../callbacks/xray_centre/ispyb_mapping.py | 1 - .../ispyb/ispyb_dataclass.py | 4 - .../external_interaction/ispyb/ispyb_store.py | 2 +- .../experiment_plans/test_fgs_plan.py | 10 +- .../external_interaction/conftest.py | 32 ++++- .../test_ispyb_dev_connection.py | 116 ++++++++++++++++-- tests/test_data/hyperion_parameters.json | 2 - .../bad_test_parameters_wrong_version.json | 2 - ...test_grid_with_edge_detect_parameters.json | 2 - .../good_test_parameters.json | 2 - ...in_centre_then_xray_centre_parameters.json | 2 - .../good_test_robot_load_params.json | 2 - ...good_test_robot_load_params_no_energy.json | 2 - .../good_test_rotation_scan_parameters.json | 2 - ..._test_rotation_scan_parameters_nomove.json | 2 - ...ood_test_stepped_grid_scan_parameters.json | 0 ...ispyb_gridscan_system_test_parameters.json | 4 +- .../live_test_rotation_params.json | 2 - .../live_test_rotation_params_move_xyz.json | 2 - .../panda_test_parameters.json | 2 - .../system_test_parameter_defaults.json | 2 - .../test_internal_parameter_defaults.json | 2 - .../test_parameter_defaults.json | 2 - .../test_parameter_defaults_2d.json | 2 - .../parameter_json_files/test_parameters.json | 2 - .../xray_centre/test_ispyb_mapping.py | 2 - .../external_interaction/ispyb/conftest.py | 2 - 27 files changed, 144 insertions(+), 63 deletions(-) create mode 100644 tests/test_data/parameter_json_files/good_test_stepped_grid_scan_parameters.json diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py index 3a5c53bde..a0c6a69a3 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py @@ -4,7 +4,6 @@ from dodal.devices.oav import utils as oav_utils from hyperion.external_interaction.ispyb.data_model import ( - DataCollectionGridInfo, DataCollectionInfo, ) diff --git a/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py b/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py index 6e5dc1073..af30524d1 100644 --- a/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py @@ -7,8 +7,6 @@ GRIDSCAN_ISPYB_PARAM_DEFAULTS = { "sample_id": None, "visit_path": "", - "microns_per_pixel_x": 0.0, - "microns_per_pixel_y": 0.0, "position": [0, 0, 0], "xtal_snapshots_omega_start": ["test_1_y", "test_2_y", "test_3_y"], "xtal_snapshots_omega_end": ["test_1_z", "test_2_z", "test_3_z"], @@ -24,8 +22,6 @@ class IspybParams(BaseModel): visit_path: str - microns_per_pixel_x: float - microns_per_pixel_y: float position: np.ndarray beam_size_x: float diff --git a/src/hyperion/external_interaction/ispyb/ispyb_store.py b/src/hyperion/external_interaction/ispyb/ispyb_store.py index a7d283443..391a74fbb 100755 --- a/src/hyperion/external_interaction/ispyb/ispyb_store.py +++ b/src/hyperion/external_interaction/ispyb/ispyb_store.py @@ -100,7 +100,7 @@ def _begin_or_update_deposition( ispyb_ids.data_collection_group_id ), "Attempt to update data collection without a data collection group ID" - grid_ids = [] + grid_ids = list(ispyb_ids.grid_ids) data_collection_ids_out = list(ispyb_ids.data_collection_ids) for scan_data_info in scan_data_infos: data_collection_id = scan_data_info.data_collection_id diff --git a/tests/system_tests/experiment_plans/test_fgs_plan.py b/tests/system_tests/experiment_plans/test_fgs_plan.py index 0b1fd4ef1..13c5fb46c 100644 --- a/tests/system_tests/experiment_plans/test_fgs_plan.py +++ b/tests/system_tests/experiment_plans/test_fgs_plan.py @@ -55,8 +55,6 @@ def params(): params = GridscanInternalParameters(**default_raw_params()) params.hyperion_params.beamline = CONST.SIM.BEAMLINE params.hyperion_params.zocalo_environment = "dev_artemis" - params.hyperion_params.ispyb_params.microns_per_pixel_x = 10 - params.hyperion_params.ispyb_params.microns_per_pixel_y = 10 yield params @@ -76,9 +74,11 @@ def reset_positions(smargon: Smargon): @pytest_asyncio.fixture async def fxc_composite(): - with patch("dodal.devices.zocalo.zocalo_results._get_zocalo_connection"), patch( - "dodal.devices.zocalo.zocalo_results.workflows.recipe" - ), patch("dodal.devices.zocalo.zocalo_results.workflows.recipe"): + with ( + patch("dodal.devices.zocalo.zocalo_results._get_zocalo_connection"), + patch("dodal.devices.zocalo.zocalo_results.workflows.recipe"), + patch("dodal.devices.zocalo.zocalo_results.workflows.recipe"), + ): zocalo = i03.zocalo() composite = FlyScanXRayCentreComposite( diff --git a/tests/system_tests/external_interaction/conftest.py b/tests/system_tests/external_interaction/conftest.py index efff3bd56..924ba59ff 100644 --- a/tests/system_tests/external_interaction/conftest.py +++ b/tests/system_tests/external_interaction/conftest.py @@ -1,11 +1,11 @@ import os from functools import partial -from typing import Callable +from typing import Any, Callable import dodal.devices.zocalo.zocalo_interaction import ispyb.sqlalchemy import pytest -from ispyb.sqlalchemy import DataCollection, DataCollectionGroup +from ispyb.sqlalchemy import DataCollection, DataCollectionGroup, GridInfo, Position from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -83,6 +83,24 @@ def get_current_datacollection_attribute( return data +def get_current_datacollection_grid_attribute( + Session: Callable, grid_id: int, attr: str +) -> Any: + with Session() as session: + query = session.query(GridInfo).filter(GridInfo.gridInfoId == grid_id) + first_result = query.first() + return getattr(first_result, attr) + + +def get_current_position_attribute( + Session: Callable, position_id: int, attr: str +) -> Any: + with Session() as session: + query = session.query(Position).filter(Position.positionId == position_id) + first_result = query.first() + return getattr(first_result, attr) + + def get_current_datacollectiongroup_attribute( Session: Callable, dcg_id: int, attr: str ): @@ -111,6 +129,16 @@ def fetch_datacollection_attribute(sqlalchemy_sessionmaker) -> Callable: return partial(get_current_datacollection_attribute, sqlalchemy_sessionmaker) +@pytest.fixture +def fetch_datacollection_grid_attribute(sqlalchemy_sessionmaker) -> Callable: + return partial(get_current_datacollection_grid_attribute, sqlalchemy_sessionmaker) + + +@pytest.fixture +def fetch_datacollection_position_attribute(sqlalchemy_sessionmaker) -> Callable: + return partial(get_current_position_attribute, sqlalchemy_sessionmaker) + + @pytest.fixture def fetch_datacollectiongroup_attribute(sqlalchemy_sessionmaker) -> Callable: return partial(get_current_datacollectiongroup_attribute, sqlalchemy_sessionmaker) diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index 821e6724e..b7112adca 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -3,9 +3,11 @@ import os import re from copy import deepcopy +from decimal import Decimal from typing import Any, Callable, Literal, Sequence from unittest.mock import MagicMock, patch +import numpy import pytest from bluesky.run_engine import RunEngine from dodal.devices.attenuator import Attenuator @@ -173,6 +175,34 @@ ] } +GRID_INFO_COLUMN_MAP = { + s.lower(): s + for s in [ + "gridInfoId", + "dataCollectionGroupId", + "xOffset", + "yOffset", + "dx_mm", + "dy_mm", + "steps_x", + "steps_y", + "meshAngle", + "pixelsPerMicronX", + "pixelsPerMicronY", + "snapshot_offsetXPixel", + "snapshot_offsetYPixel", + "recordTimeStamp", + "orientation", + "workflowMeshId", + "snaked", + "dataCollectionId", + "patchesX", + "patchesY", + "micronsPerPixelX", + "micronsPerPixelY", + ] +} + @pytest.fixture def dummy_data_collection_group_info(dummy_params): @@ -347,15 +377,14 @@ def scan_xy_data_info_for_update( ): scan_data_info_for_update = deepcopy(scan_data_info_for_begin) scan_data_info_for_update.data_collection_info.parent_id = data_collection_group_id - assert dummy_params.hyperion_params.ispyb_params is not None assert dummy_params is not None scan_data_info_for_update.data_collection_grid_info = DataCollectionGridInfo( dx_in_mm=dummy_params.experiment_params.x_step_size, dy_in_mm=dummy_params.experiment_params.y_step_size, steps_x=dummy_params.experiment_params.x_steps, steps_y=dummy_params.experiment_params.y_steps, - microns_per_pixel_x=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x, - microns_per_pixel_y=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y, + microns_per_pixel_x=1.25, + microns_per_pixel_y=1.25, # cast coordinates from numpy int64 to avoid mysql type conversion issues snapshot_offset_x_pixel=100, snapshot_offset_y_pixel=100, @@ -389,8 +418,8 @@ def scan_data_infos_for_update_3d( dy_in_mm=dummy_params.experiment_params.z_step_size, steps_x=dummy_params.experiment_params.x_steps, steps_y=dummy_params.experiment_params.z_steps, - microns_per_pixel_x=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x, - microns_per_pixel_y=dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y, + microns_per_pixel_x=1.25, + microns_per_pixel_y=1.25, # cast coordinates from numpy int64 to avoid mysql type conversion issues snapshot_offset_x_pixel=100, snapshot_offset_y_pixel=50, @@ -571,6 +600,8 @@ def test_ispyb_deposition_in_gridscan( grid_detect_then_xray_centre_composite: GridDetectThenXRayCentreComposite, grid_detect_then_xray_centre_parameters: GridScanWithEdgeDetectInternalParameters, fetch_datacollection_attribute: Callable[..., Any], + fetch_datacollection_grid_attribute: Callable[..., Any], + fetch_datacollection_position_attribute: Callable[..., Any], ): os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG grid_detect_then_xray_centre_composite.s4_slit_gaps.xgap.user_readback.sim_put(0.1) @@ -624,12 +655,41 @@ def test_ispyb_deposition_in_gridscan( ispyb_ids.data_collection_ids[0], "Hyperion: Xray centring - Diffraction grid scan of 20 by 12 " "images in 20.0 um by 20.0 um steps. Top left (px): [100,161], " - "bottom right (px): [239,245].", + "bottom right (px): [239,244].", ) compare_actual_and_expected( ispyb_ids.data_collection_ids[0], expected_values, fetch_datacollection_attribute, + DATA_COLLECTION_COLUMN_MAP, + ) + expected_values = { + "gridInfoId": ispyb_ids.grid_ids[0], + "dx_mm": 0.02, + "dy_mm": 0.02, + "steps_x": 20, + "steps_y": 12, + "snapshot_offsetXPixel": 100, + "snapshot_offsetYPixel": 161, + "orientation": "horizontal", + "snaked": True, + "dataCollectionId": ispyb_ids.data_collection_ids[0], + "micronsPerPixelX": 2.87, + "micronsPerPixelY": 2.87, + } + + compare_actual_and_expected( + ispyb_ids.grid_ids[0], + expected_values, + fetch_datacollection_grid_attribute, + GRID_INFO_COLUMN_MAP, + ) + position_id = fetch_datacollection_attribute( + ispyb_ids.data_collection_ids[0], DATA_COLLECTION_COLUMN_MAP["positionid"] + ) + expected_values = {"posX": 10.0, "posY": 20.0, "posZ": 30.0} + compare_actual_and_expected( + position_id, expected_values, fetch_datacollection_position_attribute ) expected_values = { "detectorid": 78, @@ -669,6 +729,7 @@ def test_ispyb_deposition_in_gridscan( ispyb_ids.data_collection_ids[1], expected_values, fetch_datacollection_attribute, + DATA_COLLECTION_COLUMN_MAP, ) compare_comment( fetch_datacollection_attribute, @@ -677,6 +738,33 @@ def test_ispyb_deposition_in_gridscan( "images in 20.0 um by 20.0 um steps. Top left (px): [100,165], " "bottom right (px): [239,241].", ) + position_id = fetch_datacollection_attribute( + ispyb_ids.data_collection_ids[1], DATA_COLLECTION_COLUMN_MAP["positionid"] + ) + expected_values = {"posX": 10.0, "posY": 20.0, "posZ": 30.0} + compare_actual_and_expected( + position_id, expected_values, fetch_datacollection_position_attribute + ) + expected_values = { + "gridInfoId": ispyb_ids.grid_ids[1], + "dx_mm": 0.02, + "dy_mm": 0.02, + "steps_x": 20, + "steps_y": 11, + "snapshot_offsetXPixel": 100, + "snapshot_offsetYPixel": 165, + "orientation": "horizontal", + "snaked": True, + "dataCollectionId": ispyb_ids.data_collection_ids[1], + "micronsPerPixelX": 2.87, + "micronsPerPixelY": 2.87, + } + compare_actual_and_expected( + ispyb_ids.grid_ids[1], + expected_values, + fetch_datacollection_grid_attribute, + GRID_INFO_COLUMN_MAP, + ) def compare_comment( @@ -690,10 +778,20 @@ def compare_comment( assert truncated_comment == expected_comment -def compare_actual_and_expected(dcid, expected_values, fetch_datacollection_attribute): +def compare_actual_and_expected( + id, expected_values, fetch_datacollection_attribute, column_map: dict | None = None +): for k, v in expected_values.items(): - actual = fetch_datacollection_attribute(dcid, DATA_COLLECTION_COLUMN_MAP[k]) - assert actual == v, f"expected {k} {v} == {actual}" + actual = fetch_datacollection_attribute( + id, column_map[k.lower()] if column_map else k + ) + if isinstance(actual, Decimal): + actual = float(actual) + if isinstance(v, float): + actual_v = actual == pytest.approx(v) + else: + actual_v = actual == v # if this is inlined, I don't get a nice message :/ + assert actual_v, f"expected {k} {v} == {actual}" @pytest.mark.s03 diff --git a/tests/test_data/hyperion_parameters.json b/tests/test_data/hyperion_parameters.json index 617c11404..997ffabd4 100644 --- a/tests/test_data/hyperion_parameters.json +++ b/tests/test_data/hyperion_parameters.json @@ -22,8 +22,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json b/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json index 56491115f..a91ca7ec3 100644 --- a/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json +++ b/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json @@ -20,8 +20,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json b/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json index 167b83dc0..3eb677da1 100644 --- a/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json @@ -16,8 +16,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_parameters.json b/tests/test_data/parameter_json_files/good_test_parameters.json index 55f83016a..325452603 100644 --- a/tests/test_data/parameter_json_files/good_test_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_parameters.json @@ -18,8 +18,6 @@ "sample_id": "123456", "visit_path": "/tmp/dls/i03/data/2024/cm31105-4", "current_energy_ev": 100, - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json b/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json index 86275fba4..12a10be25 100644 --- a/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json @@ -16,8 +16,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_robot_load_params.json b/tests/test_data/parameter_json_files/good_test_robot_load_params.json index d72f66990..45fc64c57 100644 --- a/tests/test_data/parameter_json_files/good_test_robot_load_params.json +++ b/tests/test_data/parameter_json_files/good_test_robot_load_params.json @@ -14,8 +14,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4", - "microns_per_pixel_x": 0.0, - "microns_per_pixel_y": 0.0, "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json b/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json index 7812a1c69..611e2a4f5 100644 --- a/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json +++ b/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json @@ -14,8 +14,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4", - "microns_per_pixel_x": 0.0, - "microns_per_pixel_y": 0.0, "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json index 8cbdfd598..3fa15c360 100644 --- a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json @@ -16,8 +16,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json index cf848a494..717e2f062 100644 --- a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json +++ b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json @@ -16,8 +16,6 @@ }, "ispyb_params": { "visit_path": "/tmp/dls/i03/data/2024/cm31105-4", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "sample_id": "123456", "position": [ 10.0, diff --git a/tests/test_data/parameter_json_files/good_test_stepped_grid_scan_parameters.json b/tests/test_data/parameter_json_files/good_test_stepped_grid_scan_parameters.json new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json index e6b4930b5..5dc833989 100644 --- a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json +++ b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json @@ -37,9 +37,7 @@ "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", - "resolution": 1.0, - "microns_per_pixel_x": -1, - "microns_per_pixel_y": -1 + "resolution": 1.0 } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/live_test_rotation_params.json b/tests/test_data/parameter_json_files/live_test_rotation_params.json index 13e13c050..ccf1c8400 100644 --- a/tests/test_data/parameter_json_files/live_test_rotation_params.json +++ b/tests/test_data/parameter_json_files/live_test_rotation_params.json @@ -16,8 +16,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json b/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json index f8a84c748..83a05bc04 100644 --- a/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json +++ b/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json @@ -16,8 +16,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/panda_test_parameters.json b/tests/test_data/parameter_json_files/panda_test_parameters.json index 2b321e878..6c23c53e2 100644 --- a/tests/test_data/parameter_json_files/panda_test_parameters.json +++ b/tests/test_data/parameter_json_files/panda_test_parameters.json @@ -17,8 +17,6 @@ "ispyb_params": { "expected_energy_ev": 12700, "visit_path": "/dls/i03/data/2023/cm33866-5/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/parameter_json_files/system_test_parameter_defaults.json b/tests/test_data/parameter_json_files/system_test_parameter_defaults.json index d59470942..f618e8961 100644 --- a/tests/test_data/parameter_json_files/system_test_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/system_test_parameter_defaults.json @@ -15,8 +15,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4", - "microns_per_pixel_x": 1.25, - "microns_per_pixel_y": 1.25, "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json b/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json index 48c03bd87..714f2d1e0 100644 --- a/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json @@ -16,8 +16,6 @@ "ispyb_params": { "expected_energy_ev": 100, "visit_path": "/tmp/cm31105-4", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_parameter_defaults.json b/tests/test_data/parameter_json_files/test_parameter_defaults.json index 2ee9dd7d6..a81c54e38 100644 --- a/tests/test_data/parameter_json_files/test_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/test_parameter_defaults.json @@ -15,8 +15,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4", - "microns_per_pixel_x": 1.25, - "microns_per_pixel_y": 1.25, "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json b/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json index e46d69a69..b18b1e590 100644 --- a/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json +++ b/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json @@ -15,8 +15,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4", - "microns_per_pixel_x": 10.0, - "microns_per_pixel_y": 10.0, "position": [ 0, 0, diff --git a/tests/test_data/parameter_json_files/test_parameters.json b/tests/test_data/parameter_json_files/test_parameters.json index aa373eb83..3f1434657 100644 --- a/tests/test_data/parameter_json_files/test_parameters.json +++ b/tests/test_data/parameter_json_files/test_parameters.json @@ -16,8 +16,6 @@ }, "ispyb_params": { "visit_path": "/tmp/cm31105-4/", - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py index 295a32078..7bc098856 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py @@ -21,8 +21,6 @@ def dummy_params(): dummy_params = GridscanInternalParameters(**default_raw_params()) dummy_params.hyperion_params.ispyb_params.sample_id = TEST_SAMPLE_ID - dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x = 1.25 - dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y = 1.25 dummy_params.hyperion_params.detector_params.run_number = 0 return dummy_params diff --git a/tests/unit_tests/external_interaction/ispyb/conftest.py b/tests/unit_tests/external_interaction/ispyb/conftest.py index 142dfecc0..835d060a3 100644 --- a/tests/unit_tests/external_interaction/ispyb/conftest.py +++ b/tests/unit_tests/external_interaction/ispyb/conftest.py @@ -27,8 +27,6 @@ def dummy_params(): dummy_params = GridscanInternalParameters(**default_raw_params()) dummy_params.hyperion_params.ispyb_params.sample_id = TEST_SAMPLE_ID - dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x = 1.25 - dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y = 1.25 dummy_params.hyperion_params.detector_params.run_number = 0 return dummy_params From f4321309d71bf359215b20365f577121743101c4 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 12 Apr 2024 10:06:34 +0100 Subject: [PATCH 10/18] (#1283) Automatically apply the snapshot microns_per_pixel when the snapshot is triggered instead of requiring the plan to apply it manually. --- .../experiment_plans/oav_grid_detection_plan.py | 7 ------- .../test_ispyb_dev_connection.py | 17 +++++++++-------- .../callbacks/xray_centre/test_ispyb_mapping.py | 2 +- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/hyperion/experiment_plans/oav_grid_detection_plan.py b/src/hyperion/experiment_plans/oav_grid_detection_plan.py index f79e7d8e4..4b6a6b4e8 100644 --- a/src/hyperion/experiment_plans/oav_grid_detection_plan.py +++ b/src/hyperion/experiment_plans/oav_grid_detection_plan.py @@ -150,19 +150,12 @@ def grid_detection_plan( yield from bps.abs_set(oav.snapshot.box_width, box_size_x_pixels) yield from bps.abs_set(oav.snapshot.num_boxes_x, x_steps) yield from bps.abs_set(oav.snapshot.num_boxes_y, y_steps) - yield from bps.abs_set( - oav.snapshot.microns_per_pixel_x, oav.parameters.micronsPerXPixel - ) - yield from bps.abs_set( - oav.snapshot.microns_per_pixel_y, oav.parameters.micronsPerYPixel - ) snapshot_filename = snapshot_template.format(angle=abs(angle)) yield from bps.abs_set(oav.snapshot.filename, snapshot_filename) yield from bps.abs_set(oav.snapshot.directory, snapshot_dir) yield from bps.trigger(oav.snapshot, wait=True) - yield from bps.create(CONST.DESCRIPTORS.OAV_SNAPSHOT_TRIGGERED) yield from bps.read(oav.snapshot) diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index b7112adca..6b82faa96 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -293,9 +293,6 @@ def grid_detect_then_xray_centre_composite( oav.snapshot.top_left_x.set(50) oav.snapshot.top_left_y.set(100) oav.snapshot.box_width.set(0.1 * 1000 / 1.25) # size in pixels - oav.snapshot.last_path_full_overlay.set("test_1_y") - oav.snapshot.last_path_outer.set("test_2_y") - oav.snapshot.last_saved_path.set("test_3_y") undulator.current_gap.sim_put(1.11) unpatched_method = oav.parameters.load_microns_per_pixel @@ -303,6 +300,12 @@ def grid_detect_then_xray_centre_composite( eiger.odin.meta.ready.sim_put(1) eiger.odin.fan.ready.sim_put(1) + def mock_snapshot_trigger(): + oav.snapshot.last_path_full_overlay.set("test_1_y") + oav.snapshot.last_path_outer.set("test_2_y") + oav.snapshot.last_saved_path.set("test_3_y") + return Status(success=True, done=True) + def patch_lmpp(zoom, xsize, ysize): unpatched_method(zoom, 1024, 768) @@ -355,9 +358,7 @@ def mock_complete_status(): "wait_for_tip_to_be_found", side_effect=mock_pin_tip_detect, ), - patch.object( - oav.snapshot, "trigger", return_value=Status(success=True, done=True) - ), + patch.object(oav.snapshot, "trigger", side_effect=mock_snapshot_trigger), patch.object( eiger.odin.file_writer.file_name, "set", @@ -604,8 +605,8 @@ def test_ispyb_deposition_in_gridscan( fetch_datacollection_position_attribute: Callable[..., Any], ): os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG - grid_detect_then_xray_centre_composite.s4_slit_gaps.xgap.user_readback.sim_put(0.1) - grid_detect_then_xray_centre_composite.s4_slit_gaps.ygap.user_readback.sim_put(0.1) + grid_detect_then_xray_centre_composite.s4_slit_gaps.xgap.user_readback.sim_put(0.1) # type: ignore + grid_detect_then_xray_centre_composite.s4_slit_gaps.ygap.user_readback.sim_put(0.1) # type: ignore ispyb_callback = GridscanISPyBCallback() RE.subscribe(ispyb_callback) RE( diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py index 7bc098856..df8d2b7fb 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py @@ -38,7 +38,7 @@ def test_ispyb_deposition_rounds_position_to_int( 20, 1.25, 1.25, - 0.01, + 0.01, # type: ignore 100, Orientation.HORIZONTAL, True, # type: ignore From 18bc8b28cbc1912911c7e356cb19ab57e388cb9c Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 12 Apr 2024 10:20:51 +0100 Subject: [PATCH 11/18] (#1283) Make ruff happy --- .../external_interaction/callbacks/xray_centre/ispyb_mapping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py index a0c6a69a3..3a5c53bde 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_mapping.py @@ -4,6 +4,7 @@ from dodal.devices.oav import utils as oav_utils from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionGridInfo, DataCollectionInfo, ) From 331c3ff5b3071dd23fb876ae64a9b2a221b14fb6 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 12 Apr 2024 12:21:28 +0100 Subject: [PATCH 12/18] (#1283) Adjust x- and y- mpp so that they are different in tests. bump dodal hash --- .../external_interaction/callbacks/conftest.py | 4 ++-- .../callbacks/xray_centre/test_ispyb_callback.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit_tests/external_interaction/callbacks/conftest.py b/tests/unit_tests/external_interaction/callbacks/conftest.py index 54fa38cf4..358b76b9a 100644 --- a/tests/unit_tests/external_interaction/callbacks/conftest.py +++ b/tests/unit_tests/external_interaction/callbacks/conftest.py @@ -162,7 +162,7 @@ class TestData: "oav_snapshot_num_boxes_x": 40, "oav_snapshot_num_boxes_y": 20, "oav_snapshot_microns_per_pixel_x": 1.25, - "oav_snapshot_microns_per_pixel_y": 1.25, + "oav_snapshot_microns_per_pixel_y": 1.5, "oav_snapshot_box_width": 0.1 * 1000 / 1.25, # size in pixels "oav_snapshot_last_path_full_overlay": "test_1_y", "oav_snapshot_last_path_outer": "test_2_y", @@ -181,7 +181,7 @@ class TestData: "oav_snapshot_num_boxes_x": 40, "oav_snapshot_num_boxes_y": 10, "oav_snapshot_microns_per_pixel_x": 1.25, - "oav_snapshot_microns_per_pixel_y": 1.25, + "oav_snapshot_microns_per_pixel_y": 1.5, "oav_snapshot_box_width": 0.1 * 1000 / 1.25, # size in pixels "oav_snapshot_last_path_full_overlay": "test_1_z", "oav_snapshot_last_path_outer": "test_2_z", diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py index e37e057b8..474c6318d 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py @@ -261,7 +261,7 @@ def test_activity_gated_event_oav_snapshot_triggered(self, mock_ispyb_conn): "xtal_snapshot2": "test_2_y", "xtal_snapshot3": "test_3_y", "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " + "images in 100.0 um by 120.0 um steps. Top left (px): [50,100], " "bottom right (px): [3250,1700].", }, ) @@ -276,7 +276,7 @@ def test_activity_gated_event_oav_snapshot_triggered(self, mock_ispyb_conn): "xtal_snapshot2": "test_2_z", "xtal_snapshot3": "test_3_z", "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 10 " - "images in 100.0 um by 100.0 um steps. Top left (px): [50,0], " + "images in 100.0 um by 120.0 um steps. Top left (px): [50,0], " "bottom right (px): [3250,800].", }, ) @@ -286,11 +286,11 @@ def test_activity_gated_event_oav_snapshot_triggered(self, mock_ispyb_conn): { "parentid": TEST_DATA_COLLECTION_IDS[0], "dxinmm": 0.1, - "dyinmm": 0.1, + "dyinmm": 0.12, "stepsx": 40, "stepsy": 20, "micronsperpixelx": 1.25, - "micronsperpixely": 1.25, + "micronsperpixely": 1.5, "snapshotoffsetxpixel": 50, "snapshotoffsetypixel": 100, "orientation": "horizontal", @@ -303,11 +303,11 @@ def test_activity_gated_event_oav_snapshot_triggered(self, mock_ispyb_conn): { "parentid": TEST_DATA_COLLECTION_IDS[1], "dxinmm": 0.1, - "dyinmm": 0.1, + "dyinmm": 0.12, "stepsx": 40, "stepsy": 10, "micronsperpixelx": 1.25, - "micronsperpixely": 1.25, + "micronsperpixely": 1.5, "snapshotoffsetxpixel": 50, "snapshotoffsetypixel": 0, "orientation": "horizontal", From 709859a137928424c651bcaa4321b23352f0dbc0 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Fri, 26 Apr 2024 13:29:40 +0100 Subject: [PATCH 13/18] (#1283) Remove microns_per_pixel from new-style parameters * Fix broken system+unit tests * Fix pathlib.Path being passed into grid_detection_plan instead of str --- .../grid_detect_then_xray_centre_plan.py | 5 +- .../callbacks/xray_centre/ispyb_callback.py | 4 +- src/hyperion/parameters/components.py | 2 - src/hyperion/parameters/gridscan.py | 2 - src/hyperion/parameters/rotation.py | 2 - .../test_ispyb_dev_connection.py | 18 +++---- ...test_grid_with_edge_detect_parameters.json | 2 - .../good_test_parameters.json | 2 - ...in_centre_then_xray_centre_parameters.json | 2 - ..._test_rotation_scan_parameters_nomove.json | 2 - ...ispyb_gridscan_system_test_parameters.json | 43 +++++++++++++++ ...ispyb_gridscan_system_test_parameters.json | 53 ------------------- .../parameters/test_parameter_model.py | 8 --- 13 files changed, 55 insertions(+), 90 deletions(-) create mode 100644 tests/test_data/new_parameter_json_files/ispyb_gridscan_system_test_parameters.json delete mode 100644 tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json diff --git a/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py b/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py index 3a1571f7d..446293cf2 100644 --- a/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py @@ -2,6 +2,7 @@ import dataclasses import json +from pathlib import Path from blueapi.core import BlueskyContext, MsgGenerator from bluesky import plan_stubs as bps @@ -156,7 +157,7 @@ def _detect_grid_and_do_gridscan( def run_grid_detection_plan( oav_params, snapshot_template, - snapshot_dir, + snapshot_dir: Path, ): grid_detect_composite = OavGridDetectionComposite( backlight=composite.backlight, @@ -169,7 +170,7 @@ def run_grid_detection_plan( grid_detect_composite, oav_params, snapshot_template, - snapshot_dir, + str(snapshot_dir), grid_width_microns=parameters.grid_width_um, ) diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py index 7d4cab607..1dfcbf1dd 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py @@ -32,7 +32,6 @@ ) from hyperion.log import ISPYB_LOGGER, set_dcgid_tag from hyperion.parameters.constants import CONST -from hyperion.parameters.gridscan import ThreeDGridScan from hyperion.parameters.plan_specific.gridscan_internal_params import ( GridscanInternalParameters, ) @@ -49,7 +48,7 @@ def ispyb_activation_wrapper(plan_generator, parameters): "subplan_name": CONST.PLAN.GRID_DETECT_AND_DO_GRIDSCAN, "hyperion_internal_parameters": ( parameters.old_parameters() - if isinstance(parameters, ThreeDGridScan) + if callable(getattr(parameters, "old_parameters", 0)) else parameters ).json(), }, @@ -118,7 +117,6 @@ def activity_gated_start(self, doc: RunStart): None, None, populate_xz_data_collection_info( - self.params.hyperion_params.detector_params ), self.params.hyperion_params.detector_params, diff --git a/src/hyperion/parameters/components.py b/src/hyperion/parameters/components.py index 7e7eeb73a..0ed145800 100644 --- a/src/hyperion/parameters/components.py +++ b/src/hyperion/parameters/components.py @@ -194,8 +194,6 @@ class Config: arbitrary_types_allowed = True extra = Extra.forbid - microns_per_pixel_x: float - microns_per_pixel_y: float position: list[float] | NDArray = Field(default=np.array([0, 0, 0])) beam_size_x: float beam_size_y: float diff --git a/src/hyperion/parameters/gridscan.py b/src/hyperion/parameters/gridscan.py index 8da34a3ac..32bd49b14 100644 --- a/src/hyperion/parameters/gridscan.py +++ b/src/hyperion/parameters/gridscan.py @@ -65,8 +65,6 @@ class GridCommon( def ispyb_params(self): return GridscanIspybParams( visit_path=str(self.visit_directory), - microns_per_pixel_x=self.ispyb_extras.microns_per_pixel_x, - microns_per_pixel_y=self.ispyb_extras.microns_per_pixel_y, position=np.array(self.ispyb_extras.position), beam_size_x=self.ispyb_extras.beam_size_x, beam_size_y=self.ispyb_extras.beam_size_y, diff --git a/src/hyperion/parameters/rotation.py b/src/hyperion/parameters/rotation.py index 82f90ecde..4461cbfd8 100644 --- a/src/hyperion/parameters/rotation.py +++ b/src/hyperion/parameters/rotation.py @@ -82,8 +82,6 @@ def detector_params(self): def ispyb_params(self): # pyright: ignore return RotationIspybParams( visit_path=str(self.visit_directory), - microns_per_pixel_x=self.ispyb_extras.microns_per_pixel_x, - microns_per_pixel_y=self.ispyb_extras.microns_per_pixel_y, position=np.array(self.ispyb_extras.position), beam_size_x=self.ispyb_extras.beam_size_x, beam_size_y=self.ispyb_extras.beam_size_y, diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index 6b82faa96..b923f96b0 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -54,9 +54,7 @@ StoreInIspyb, ) from hyperion.parameters.constants import CONST -from hyperion.parameters.plan_specific.grid_scan_with_edge_detect_params import ( - GridScanWithEdgeDetectInternalParameters, -) +from hyperion.parameters.gridscan import GridScanWithEdgeDetect from hyperion.parameters.plan_specific.rotation_scan_internal_params import ( RotationInternalParameters, ) @@ -233,9 +231,9 @@ def dummy_scan_data_info_for_begin(dummy_params): @pytest.fixture def grid_detect_then_xray_centre_parameters(): json_dict = raw_params_from_file( - "tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json" + "tests/test_data/new_parameter_json_files/ispyb_gridscan_system_test_parameters.json" ) - return GridScanWithEdgeDetectInternalParameters(**json_dict) + return GridScanWithEdgeDetect(**json_dict) # noinspection PyUnreachableCode @@ -599,7 +597,7 @@ def generate_scan_data_infos( def test_ispyb_deposition_in_gridscan( RE: RunEngine, grid_detect_then_xray_centre_composite: GridDetectThenXRayCentreComposite, - grid_detect_then_xray_centre_parameters: GridScanWithEdgeDetectInternalParameters, + grid_detect_then_xray_centre_parameters: GridScanWithEdgeDetect, fetch_datacollection_attribute: Callable[..., Any], fetch_datacollection_grid_attribute: Callable[..., Any], fetch_datacollection_position_attribute: Callable[..., Any], @@ -629,7 +627,7 @@ def test_ispyb_deposition_in_gridscan( "beamsizeatsamplex": 1, "beamsizeatsampley": 1, "transmission": 49.118, - "datacollectionnumber": 0, + "datacollectionnumber": 1, "detectordistance": 100.0, "exposuretime": 0.12, "imagedirectory": "/tmp/", @@ -648,7 +646,7 @@ def test_ispyb_deposition_in_gridscan( "xtalsnapshotfullpath3": "test_3_y", "synchrotronmode": "User", "undulatorgap1": 1.11, - "filetemplate": "file_name_0_master.h5", + "filetemplate": "file_name_1_master.h5", "numberofimages": 20 * 12, } compare_comment( @@ -704,7 +702,7 @@ def test_ispyb_deposition_in_gridscan( "beamsizeatsamplex": 1, "beamsizeatsampley": 1, "transmission": 49.118, - "datacollectionnumber": 1, + "datacollectionnumber": 2, "detectordistance": 100.0, "exposuretime": 0.12, "imagedirectory": "/tmp/", @@ -723,7 +721,7 @@ def test_ispyb_deposition_in_gridscan( "xtalsnapshotfullpath3": "test_3_y", "synchrotronmode": "User", "undulatorgap1": 1.11, - "filetemplate": "file_name_1_master.h5", + "filetemplate": "file_name_2_master.h5", "numberofimages": 20 * 11, } compare_actual_and_expected( diff --git a/tests/test_data/new_parameter_json_files/good_test_grid_with_edge_detect_parameters.json b/tests/test_data/new_parameter_json_files/good_test_grid_with_edge_detect_parameters.json index d7f6fd7e8..7aa7aa583 100644 --- a/tests/test_data/new_parameter_json_files/good_test_grid_with_edge_detect_parameters.json +++ b/tests/test_data/new_parameter_json_files/good_test_grid_with_edge_detect_parameters.json @@ -18,8 +18,6 @@ "transmission_frac": 1.0, "visit": "cm31105-4", "ispyb_extras": { - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/new_parameter_json_files/good_test_parameters.json b/tests/test_data/new_parameter_json_files/good_test_parameters.json index b70d1bfff..63b37b24c 100644 --- a/tests/test_data/new_parameter_json_files/good_test_parameters.json +++ b/tests/test_data/new_parameter_json_files/good_test_parameters.json @@ -27,8 +27,6 @@ "z2_start_um": 0.0, "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/xraycentring/123456/", "ispyb_extras": { - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/new_parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json b/tests/test_data/new_parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json index b87402df9..5409e8c38 100644 --- a/tests/test_data/new_parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json +++ b/tests/test_data/new_parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json @@ -19,8 +19,6 @@ "transmission_frac": 1.0, "visit": "cm31105-4", "ispyb_extras": { - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/new_parameter_json_files/good_test_rotation_scan_parameters_nomove.json b/tests/test_data/new_parameter_json_files/good_test_rotation_scan_parameters_nomove.json index 2889a6d57..4ac049bbf 100644 --- a/tests/test_data/new_parameter_json_files/good_test_rotation_scan_parameters_nomove.json +++ b/tests/test_data/new_parameter_json_files/good_test_rotation_scan_parameters_nomove.json @@ -21,8 +21,6 @@ "zocalo_environment": "dev_artemis", "transmission_frac": 0.1, "ispyb_extras": { - "microns_per_pixel_x": 1.0, - "microns_per_pixel_y": 1.0, "position": [ 10.0, 20.0, diff --git a/tests/test_data/new_parameter_json_files/ispyb_gridscan_system_test_parameters.json b/tests/test_data/new_parameter_json_files/ispyb_gridscan_system_test_parameters.json new file mode 100644 index 000000000..3a2b36464 --- /dev/null +++ b/tests/test_data/new_parameter_json_files/ispyb_gridscan_system_test_parameters.json @@ -0,0 +1,43 @@ +{ + "parameter_model_version": "5.0.0", + "beamline": "BL03S", + "insertion_prefix": "SR03S", + "detector": "EIGER2_X_16M", + "zocalo_environment": "dev_artemis", + "storage_directory": "/tmp", + "file_name": "file_name", + "run_number": 0, + "sample_id": 123456, + "use_roi_mode": false, + "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt", + "exposure_time_s": 0.12, + "detector_distance_mm": 100.0, + "omega_start_deg": 0.0, + "grid_width_um": 400, + "oav_centring_file": "tests/test_data/test_OAVCentring.json", + "transmission_frac": 1.0, + "visit": "cm31105-4", + "demand_energy_ev": 12700, + "ispyb_extras": { + "position": [ + 10.0, + 20.0, + 30.0 + ], + "xtal_snapshots_omega_start": [ + "test_1_y", + "test_2_y", + "test_3_y" + ], + "xtal_snapshots_omega_end": [ + "test_1_z", + "test_2_z", + "test_3_z" + ], + "beam_size_x": 1.0, + "beam_size_y": 1.0, + "focal_spot_size_x": 1.0, + "focal_spot_size_y": 1.0, + "resolution": 1.0 + } +} \ No newline at end of file diff --git a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json b/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json deleted file mode 100644 index 5dc833989..000000000 --- a/tests/test_data/parameter_json_files/ispyb_gridscan_system_test_parameters.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "params_version": "4.0.5", - "hyperion_params": { - "beamline": "BL03S", - "insertion_prefix": "SR03S", - "detector": "EIGER2_X_16M", - "zocalo_environment": "dev_artemis", - "experiment_type": "grid_detect_then_xray_centre", - "detector_params": { - "expected_energy_ev": 100, - "directory": "/tmp", - "prefix": "file_name", - "run_number": 0, - "use_roi_mode": false, - "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt" - }, - "ispyb_params": { - "visit_path": "/tmp/cm31105-4/", - "position": [ - 10.0, - 20.0, - 30.0 - ], - "xtal_snapshots_omega_start": [ - "test_1_y", - "test_2_y", - "test_3_y" - ], - "xtal_snapshots_omega_end": [ - "test_1_z", - "test_2_z", - "test_3_z" - ], - "transmission_fraction": 1.0, - "beam_size_x": 1.0, - "beam_size_y": 1.0, - "focal_spot_size_x": 1.0, - "focal_spot_size_y": 1.0, - "comment": "test", - "resolution": 1.0 - } - }, - "experiment_params": { - "z_steps": 10, - "snapshot_dir": "/tmp", - "transmission_fraction": 1.0, - "detector_distance": 100.0, - "omega_start": 0.0, - "grid_width_microns": 400, - "exposure_time": 0.12, - "requested_energy_kev": 12.7 - } -} \ No newline at end of file diff --git a/tests/unit_tests/parameters/test_parameter_model.py b/tests/unit_tests/parameters/test_parameter_model.py index 3a5a70542..a86ee8a03 100644 --- a/tests/unit_tests/parameters/test_parameter_model.py +++ b/tests/unit_tests/parameters/test_parameter_model.py @@ -41,8 +41,6 @@ def minimal_3d_gridscan_params(): "z_steps": 9, "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/xraycentring/123456/", "ispyb_extras": { - "microns_per_pixel_x": 0, - "microns_per_pixel_y": 0, "position": [0, 0, 0], "beam_size_x": 0, "beam_size_y": 0, @@ -140,8 +138,6 @@ def test_robot_load_then_centre_params(): "file_name": "file_name", "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/xraycentring/123456/", "ispyb_extras": { - "microns_per_pixel_x": 0.5, - "microns_per_pixel_y": 0.5, "beam_size_x": 0.05, "beam_size_y": 0.05, "focal_spot_size_x": 0.06, @@ -195,8 +191,6 @@ def test_pin_then_xray(self): "zocalo_environment": "artemis", "storage_directory": self.directory, "ispyb_extras": { - "microns_per_pixel_x": self.microns_per_pixel_x, - "microns_per_pixel_y": self.microns_per_pixel_y, "position": self.position, "beam_size_x": self.beam_size_x, "beam_size_y": self.beam_size_y, @@ -284,8 +278,6 @@ def test_rotation_new_params(self): "zocalo_environment": "artemis", "transmission_frac": self.transmission, "ispyb_extras": { - "microns_per_pixel_x": self.microns_per_pixel_x, - "microns_per_pixel_y": self.microns_per_pixel_y, "xtal_snapshots_omega_start": ["test1", "test2", "test3"], "xtal_snapshots_omega_end": ["", "", ""], "position": self.position, From e8c1b9fa9e87e6a831a460c587416f1dd7d8510c Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 29 Apr 2024 09:11:08 +0100 Subject: [PATCH 14/18] (DiamondLightSource/dodal#466) changes to support directory provider in panda * Change references from PandA to HDFPanda * Apply directory path from params --- setup.cfg | 2 +- .../device_setup_plans/setup_panda.py | 8 ++--- .../flyscan_xray_centre_plan.py | 4 +-- .../grid_detect_then_xray_centre_plan.py | 4 +-- .../panda_flyscan_xray_centre_plan.py | 5 +++ .../robot_load_then_centre_plan.py | 4 +-- .../test_panda_flyscan_xray_centre_plan.py | 35 +++++++++++++++++++ 7 files changed, 51 insertions(+), 11 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7d09abc96..8b1ef4b29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ install_requires = xarray doct databroker - dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@7ecd49eb11060a8528ed8b6d31e6c94179bdc2a3 + dls-dodal @ git+https://github.com/DiamondLightSource/dodal.git@aac89a96686218add67bbbcc80118f6e94f0bdc9 pydantic<2.0 # See https://github.com/DiamondLightSource/hyperion/issues/774 scipy pyzmq<25 # See https://github.com/DiamondLightSource/hyperion/issues/1103 diff --git a/src/hyperion/device_setup_plans/setup_panda.py b/src/hyperion/device_setup_plans/setup_panda.py index 7c84d801f..6d4c64918 100644 --- a/src/hyperion/device_setup_plans/setup_panda.py +++ b/src/hyperion/device_setup_plans/setup_panda.py @@ -5,7 +5,7 @@ from dodal.devices.panda_fast_grid_scan import PandAGridScanParams from ophyd_async.core import load_device from ophyd_async.panda import ( - PandA, + HDFPanda, SeqTable, SeqTableRow, SeqTrigger, @@ -109,7 +109,7 @@ def get_seq_table( def setup_panda_for_flyscan( - panda: PandA, + panda: HDFPanda, config_yaml_path: str, parameters: PandAGridScanParams, initial_x: float, @@ -123,7 +123,7 @@ def setup_panda_for_flyscan( created using ophyd_async.core.save_device() Args: - panda (PandA): The PandA Ophyd device + panda (HDFPanda): The PandA Ophyd device config_yaml_path (str): Path to the yaml file containing the desired PandA PVs parameters (PandAGridScanParams): Grid parameters initial_x (float): Motor positions at time of PandA setup @@ -177,7 +177,7 @@ def setup_panda_for_flyscan( yield from arm_panda_for_gridscan(panda) -def arm_panda_for_gridscan(panda: PandA, group="arm_panda_gridscan"): +def arm_panda_for_gridscan(panda: HDFPanda, group="arm_panda_gridscan"): yield from bps.abs_set(panda.seq[1].enable, Enabled.ENABLED.value, group=group) # type: ignore yield from bps.abs_set(panda.pulse[1].enable, Enabled.ENABLED.value, group=group) # type: ignore yield from bps.abs_set(panda.counter[1].enable, Enabled.ENABLED.value, group=group) # type: ignore diff --git a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py index e24d57460..019f8e6fe 100755 --- a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py @@ -33,7 +33,7 @@ get_processing_result, ) from dodal.plans.check_topup import check_topup_and_wait_if_necessary -from ophyd_async.panda import PandA +from ophyd_async.panda import HDFPanda from hyperion.device_setup_plans.manipulate_sample import move_x_y_z from hyperion.device_setup_plans.read_hardware_for_setup import ( @@ -85,7 +85,7 @@ class FlyScanXRayCentreComposite: xbpm_feedback: XBPMFeedback zebra: Zebra zocalo: ZocaloResults - panda: PandA + panda: HDFPanda panda_fast_grid_scan: PandAFastGridScan robot: BartRobot diff --git a/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py b/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py index 446293cf2..69316958b 100644 --- a/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/grid_detect_then_xray_centre_plan.py @@ -27,7 +27,7 @@ from dodal.devices.xbpm_feedback import XBPMFeedback from dodal.devices.zebra import Zebra from dodal.devices.zocalo import ZocaloResults -from ophyd_async.panda import PandA +from ophyd_async.panda import HDFPanda from hyperion.device_setup_plans.utils import ( start_preparing_data_collection_then_do_plan, @@ -88,7 +88,7 @@ class GridDetectThenXRayCentreComposite: xbpm_feedback: XBPMFeedback zebra: Zebra zocalo: ZocaloResults - panda: PandA + panda: HDFPanda panda_fast_grid_scan: PandAFastGridScan robot: BartRobot diff --git a/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py index 9089a0429..0e8f421ee 100755 --- a/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py @@ -1,11 +1,13 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING, Any, Union import bluesky.plan_stubs as bps import bluesky.preprocessors as bpp import numpy as np from blueapi.core import BlueskyContext, MsgGenerator +from dodal.common import udc_directory_provider from dodal.devices.panda_fast_grid_scan import ( set_fast_grid_scan_params as set_flyscan_params, ) @@ -194,6 +196,9 @@ def run_gridscan_and_move( time_between_x_steps_ms, ) + udc_directory_provider.set_directory( + Path(parameters.hyperion_params.detector_params.directory) + ) yield from setup_panda_for_flyscan( fgs_composite.panda, PANDA_SETUP_PATH, diff --git a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py index dccd7096a..144ee73d8 100644 --- a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py +++ b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py @@ -31,7 +31,7 @@ from dodal.devices.xbpm_feedback import XBPMFeedback from dodal.devices.zebra import Zebra from dodal.devices.zocalo import ZocaloResults -from ophyd_async.panda import PandA +from ophyd_async.panda import HDFPanda from hyperion.device_setup_plans.utils import ( start_preparing_data_collection_then_do_plan, @@ -79,7 +79,7 @@ class RobotLoadThenCentreComposite: undulator: Undulator zebra: Zebra zocalo: ZocaloResults - panda: PandA + panda: HDFPanda panda_fast_grid_scan: PandAFastGridScan # SetEnergyComposite fields diff --git a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py index 3c8e87158..9cebfa132 100644 --- a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py @@ -1,5 +1,6 @@ import random import types +from pathlib import Path from typing import Any, Tuple from unittest.mock import DEFAULT, MagicMock, call, patch @@ -470,6 +471,40 @@ def wrapped_run_gridscan_and_move(): call = app_to_comment.call_args_list[0] assert "Crystal 1: Strength 999999" in call.args[1] + @patch( + "dodal.devices.aperturescatterguard.ApertureScatterguard.set", + new=MagicMock(return_value=Status(done=True, success=True)), + ) + @patch( + "hyperion.experiment_plans.panda_flyscan_xray_centre_plan.move_x_y_z", + new=MagicMock(autospec=True), + ) + @patch( + "hyperion.experiment_plans.panda_flyscan_xray_centre_plan.setup_panda_for_flyscan", + new=MagicMock(autospec=True), + ) + @patch( + "hyperion.experiment_plans.panda_flyscan_xray_centre_plan.run_gridscan", + new=MagicMock(return_value=iter([])), + ) + @patch( + "hyperion.experiment_plans.panda_flyscan_xray_centre_plan.udc_directory_provider.set_directory", + autospec=True, + ) + def test_when_gridscan_run_panda_directory_applied( + self, + set_directory, + RE_with_subs: tuple[RunEngine, Any], + test_panda_fgs_params: PandAGridscanInternalParameters, + fake_fgs_composite: FlyScanXRayCentreComposite, + ): + RE_with_subs[0].subscribe(VerbosePlanExecutionLoggingCallback()) + RE_with_subs[0]( + run_gridscan_and_move(fake_fgs_composite, test_panda_fgs_params) + ) + expected_path = Path("/dls/i03/data/2023/cm33866-5/test_hyperion") + set_directory.assert_called_once_with(expected_path) + @patch( "hyperion.experiment_plans.panda_flyscan_xray_centre_plan.run_gridscan", autospec=True, From 355cb73c8577f9a7c2c408fbf73787306dd32435 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 29 Apr 2024 14:11:57 +0100 Subject: [PATCH 15/18] (#1293) Remove sample_barcode from ispyb callbacks and from read_hardware_for_setup() --- .../device_setup_plans/read_hardware_for_setup.py | 1 - .../callbacks/common/ispyb_mapping.py | 6 +----- .../callbacks/ispyb_callback_base.py | 10 +--------- .../external_interaction/test_ispyb_dev_connection.py | 2 -- .../good_test_robot_load_params.json | 3 +-- tests/unit_tests/experiment_plans/conftest.py | 1 - .../experiment_plans/test_flyscan_xray_centre_plan.py | 2 -- .../test_panda_flyscan_xray_centre_plan.py | 3 --- .../external_interaction/callbacks/conftest.py | 1 - .../callbacks/rotation/test_ispyb_callback.py | 1 - .../callbacks/xray_centre/test_ispyb_callback.py | 1 - tests/unit_tests/parameters/test_parameter_model.py | 2 -- 12 files changed, 3 insertions(+), 30 deletions(-) diff --git a/src/hyperion/device_setup_plans/read_hardware_for_setup.py b/src/hyperion/device_setup_plans/read_hardware_for_setup.py index ca9308394..23ace4ddb 100644 --- a/src/hyperion/device_setup_plans/read_hardware_for_setup.py +++ b/src/hyperion/device_setup_plans/read_hardware_for_setup.py @@ -31,7 +31,6 @@ def read_hardware_for_ispyb_pre_collection( yield from bps.read(s4_slit_gaps.xgap) yield from bps.read(s4_slit_gaps.ygap) yield from bps.read(aperture_scatterguard) - yield from bps.read(robot.barcode) yield from bps.save() diff --git a/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py index 73b9255a5..377236657 100644 --- a/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py @@ -23,16 +23,12 @@ def populate_data_collection_group( - experiment_type: str, - detector_params: DetectorParams, - ispyb_params: IspybParams, - sample_barcode: Optional[str] = None, + experiment_type: str, detector_params: DetectorParams, ispyb_params: IspybParams ): dcg_info = DataCollectionGroupInfo( visit_string=get_visit_string(ispyb_params, detector_params), experiment_type=experiment_type, sample_id=ispyb_params.sample_id, - sample_barcode=sample_barcode, ) return dcg_info diff --git a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py index 630595098..7e92b4fcd 100644 --- a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -55,7 +55,6 @@ def __init__( ISPYB_LOGGER.debug("Initialising ISPyB callback") super().__init__(log=ISPYB_LOGGER, emit=emit) self._oav_snapshot_event_idx: int = 0 - self._sample_barcode: Optional[str] = None self.params: GridscanInternalParameters | RotationInternalParameters | None = ( None ) @@ -77,7 +76,6 @@ def __init__( def activity_gated_start(self, doc: RunStart): self._oav_snapshot_event_idx = 0 - self._sample_barcode = None return self._tag_doc(doc) def activity_gated_descriptor(self, doc: EventDescriptor): @@ -131,7 +129,6 @@ def _handle_ispyb_hardware_read( slitgap_horizontal=doc["data"]["s4_slit_gaps_xgap"], slitgap_vertical=doc["data"]["s4_slit_gaps_ygap"], ) - self._sample_barcode = doc["data"]["robot-barcode"] scan_data_infos = self.populate_info_for_update( hwscan_data_collection_info, self.params ) @@ -142,7 +139,6 @@ def _handle_ispyb_hardware_read( self.ispyb.experiment_type, self.params.hyperion_params.detector_params, self.params.hyperion_params.ispyb_params, - self._sample_barcode, ) return data_collection_group_info, scan_data_infos @@ -210,16 +206,12 @@ def _handle_ispyb_transmission_flux_read(self, doc) -> Sequence[ScanDataInfo]: return scan_data_infos def update_deposition( - self, - params, - scan_data_infos: Sequence[ScanDataInfo], - sample_barcode: Optional[str], + self, params, scan_data_infos: Sequence[ScanDataInfo] ) -> IspybIds: data_collection_group_info = populate_data_collection_group( self.ispyb.experiment_type, params.hyperion_params.detector_params, params.hyperion_params.ispyb_params, - sample_barcode, ) return self.ispyb.update_deposition( diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index b923f96b0..071b07292 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -893,5 +893,3 @@ def test_ispyb_deposition_in_rotation_plan( fetch_datacollection_attribute(dcid, "slitGapHorizontal") == test_slit_gap_horiz ) assert fetch_datacollection_attribute(dcid, "slitGapVertical") == test_slit_gap_vert - # TODO Can't test barcode as need BLSample which needs Dewar, Shipping, Container entries for the - # upsert stored proc to use it. diff --git a/tests/test_data/parameter_json_files/good_test_robot_load_params.json b/tests/test_data/parameter_json_files/good_test_robot_load_params.json index 45fc64c57..206931f51 100644 --- a/tests/test_data/parameter_json_files/good_test_robot_load_params.json +++ b/tests/test_data/parameter_json_files/good_test_robot_load_params.json @@ -24,8 +24,7 @@ "focal_spot_size_x": 0.0, "focal_spot_size_y": 0.0, "comment": "Descriptive comment.", - "sample_id": 12345, - "sample_barcode": null + "sample_id": 12345 } }, "experiment_params": { diff --git a/tests/unit_tests/experiment_plans/conftest.py b/tests/unit_tests/experiment_plans/conftest.py index 7926b8caf..14d77e80d 100644 --- a/tests/unit_tests/experiment_plans/conftest.py +++ b/tests/unit_tests/experiment_plans/conftest.py @@ -47,7 +47,6 @@ def make_event_doc(data, descriptor="abc123") -> Event: "synchrotron-synchrotron_mode": SynchrotronMode.USER, "s4_slit_gaps_xgap": 0, "s4_slit_gaps_ygap": 0, - "robot-barcode": "BARCODE", } BASIC_POST_SETUP_DOC = { diff --git a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py index bc160abf7..694efa25b 100644 --- a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py @@ -214,7 +214,6 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( fake_fgs_composite.aperture_scatterguard.set( fake_fgs_composite.aperture_scatterguard.aperture_positions.LARGE # type: ignore ) - set_sim_value(fake_fgs_composite.robot.barcode.bare_signal, ["BARCODE"]) test_ispyb_callback = PlanReactiveCallback(ISPYB_LOGGER) test_ispyb_callback.active = True @@ -253,7 +252,6 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( "synchrotron-synchrotron_mode": synchrotron_test_value.value, "s4_slit_gaps_xgap": xgap_test_value, "s4_slit_gaps_ygap": ygap_test_value, - "robot-barcode": "BARCODE", }, ) assert_event( diff --git a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py index 9cebfa132..3d793446e 100644 --- a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py @@ -161,8 +161,6 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( flux_test_value = 10.0 fake_fgs_composite.flux.flux_reading.sim_put(flux_test_value) # type: ignore - set_sim_value(fake_fgs_composite.robot.barcode.bare_signal, ["BARCODE"]) - test_ispyb_callback = PlanReactiveCallback(ISPYB_LOGGER) test_ispyb_callback.active = True with patch.multiple( @@ -199,7 +197,6 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( "synchrotron-synchrotron_mode": synchrotron_test_value.value, "s4_slit_gaps_xgap": xgap_test_value, "s4_slit_gaps_ygap": ygap_test_value, - "robot-barcode": "BARCODE", }, ) assert_event( diff --git a/tests/unit_tests/external_interaction/callbacks/conftest.py b/tests/unit_tests/external_interaction/callbacks/conftest.py index 358b76b9a..28256c5d0 100644 --- a/tests/unit_tests/external_interaction/callbacks/conftest.py +++ b/tests/unit_tests/external_interaction/callbacks/conftest.py @@ -196,7 +196,6 @@ class TestData: "s4_slit_gaps_ygap": 0.2345, "synchrotron-synchrotron_mode": SynchrotronMode.USER, "undulator_current_gap": 1.234, - "robot-barcode": "BARCODE", }, "timestamps": {"det1": 1666604299.8220396, "det2": 1666604299.8235943}, "seq_num": 1, diff --git a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py index bbf79e6ce..a5f8ad260 100644 --- a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py @@ -107,7 +107,6 @@ def test_hardware_read_events( "parentid": TEST_SESSION_ID, "experimenttype": "SAD", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": "BARCODE", }, ) assert_upsert_call_with( diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py index 474c6318d..aeb0544e7 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py @@ -142,7 +142,6 @@ def test_hardware_read_event_3d(self, mock_ispyb_conn): "parentid": TEST_SESSION_ID, "experimenttype": "Mesh3D", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": "BARCODE", # deferred }, ) assert_upsert_call_with( diff --git a/tests/unit_tests/parameters/test_parameter_model.py b/tests/unit_tests/parameters/test_parameter_model.py index a86ee8a03..ba0be3c84 100644 --- a/tests/unit_tests/parameters/test_parameter_model.py +++ b/tests/unit_tests/parameters/test_parameter_model.py @@ -221,7 +221,6 @@ def test_pin_then_xray(self): "undulator_gap": 0.5, "microns_per_pixel_x": self.microns_per_pixel_x, "microns_per_pixel_y": self.microns_per_pixel_y, - "sample_barcode": "", "position": self.position, "beam_size_x": self.beam_size_x, "beam_size_y": self.beam_size_y, @@ -313,7 +312,6 @@ def test_rotation_new_params(self): "undulator_gap": 0.5, "microns_per_pixel_x": self.microns_per_pixel_x, "microns_per_pixel_y": self.microns_per_pixel_y, - "sample_barcode": "", "position": self.position, "beam_size_x": self.beam_size_x, "beam_size_y": self.beam_size_y, From a55001bba0b58b7f25ea4a4dc614517673f8cd06 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 29 Apr 2024 15:12:22 +0100 Subject: [PATCH 16/18] (#1293) ispyb callback event handlers no longer update the DataCollectionGroup --- .../callbacks/ispyb_callback_base.py | 45 +++---------------- .../external_interaction/ispyb/ispyb_store.py | 5 +-- .../test_ispyb_dev_connection.py | 8 +--- .../callbacks/rotation/test_ispyb_callback.py | 11 +---- .../xray_centre/test_ispyb_callback.py | 11 +---- .../ispyb/test_gridscan_ispyb_store_3d.py | 40 ++++++----------- .../ispyb/test_rotation_ispyb_store.py | 28 +----------- 7 files changed, 27 insertions(+), 121 deletions(-) diff --git a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py index 7e92b4fcd..5d1210530 100644 --- a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -6,9 +6,6 @@ from dodal.devices.synchrotron import SynchrotronMode -from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( - populate_data_collection_group, -) from hyperion.external_interaction.callbacks.plan_reactive_callback import ( PlanReactiveCallback, ) @@ -17,7 +14,6 @@ ) from hyperion.external_interaction.ispyb.data_model import ( DataCollectionGridInfo, - DataCollectionGroupInfo, DataCollectionInfo, ScanDataInfo, ) @@ -96,27 +92,20 @@ def activity_gated_event(self, doc: Event) -> Event: "has no corresponding descriptor record" ) return doc - data_collection_group_info = None match event_descriptor.get("name"): case CONST.DESCRIPTORS.ISPYB_HARDWARE_READ: - data_collection_group_info, scan_data_infos = ( - self._handle_ispyb_hardware_read(doc) - ) + scan_data_infos = self._handle_ispyb_hardware_read(doc) case CONST.DESCRIPTORS.OAV_SNAPSHOT_TRIGGERED: scan_data_infos = self._handle_oav_snapshot_triggered(doc) case CONST.DESCRIPTORS.ISPYB_TRANSMISSION_FLUX_READ: scan_data_infos = self._handle_ispyb_transmission_flux_read(doc) case _: return self._tag_doc(doc) - self.ispyb_ids = self.ispyb.update_deposition( - self.ispyb_ids, data_collection_group_info, scan_data_infos - ) + self.ispyb_ids = self.ispyb.update_deposition(self.ispyb_ids, scan_data_infos) ISPYB_LOGGER.info(f"Recieved ISPYB IDs: {self.ispyb_ids}") return self._tag_doc(doc) - def _handle_ispyb_hardware_read( - self, doc - ) -> tuple[DataCollectionGroupInfo, Sequence[ScanDataInfo]]: + def _handle_ispyb_hardware_read(self, doc) -> Sequence[ScanDataInfo]: assert self.params, "Event handled before activity_gated_start received params" ISPYB_LOGGER.info("ISPyB handler received event from read hardware") assert isinstance( @@ -132,15 +121,8 @@ def _handle_ispyb_hardware_read( scan_data_infos = self.populate_info_for_update( hwscan_data_collection_info, self.params ) - ISPYB_LOGGER.info( - "Updating ispyb data collection and group after hardware read." - ) - data_collection_group_info = populate_data_collection_group( - self.ispyb.experiment_type, - self.params.hyperion_params.detector_params, - self.params.hyperion_params.ispyb_params, - ) - return data_collection_group_info, scan_data_infos + ISPYB_LOGGER.info("Updating ispyb data collection after hardware read.") + return scan_data_infos def _handle_oav_snapshot_triggered(self, doc) -> Sequence[ScanDataInfo]: assert self.ispyb_ids.data_collection_ids, "No current data collection" @@ -182,9 +164,7 @@ def _handle_oav_snapshot_triggered(self, doc) -> Sequence[ScanDataInfo]: data_collection_id=data_collection_id, data_collection_grid_info=data_collection_grid_info, ) - ISPYB_LOGGER.info( - "Updating ispyb data collection and group after oav snapshot." - ) + ISPYB_LOGGER.info("Updating ispyb data collection after oav snapshot.") self._oav_snapshot_event_idx += 1 return [scan_data_info] @@ -205,19 +185,6 @@ def _handle_ispyb_transmission_flux_read(self, doc) -> Sequence[ScanDataInfo]: ISPYB_LOGGER.info("Updating ispyb data collection after flux read.") return scan_data_infos - def update_deposition( - self, params, scan_data_infos: Sequence[ScanDataInfo] - ) -> IspybIds: - data_collection_group_info = populate_data_collection_group( - self.ispyb.experiment_type, - params.hyperion_params.detector_params, - params.hyperion_params.ispyb_params, - ) - - return self.ispyb.update_deposition( - self.ispyb_ids, data_collection_group_info, scan_data_infos - ) - @abstractmethod def populate_info_for_update( self, diff --git a/src/hyperion/external_interaction/ispyb/ispyb_store.py b/src/hyperion/external_interaction/ispyb/ispyb_store.py index 391a74fbb..80953f5ee 100755 --- a/src/hyperion/external_interaction/ispyb/ispyb_store.py +++ b/src/hyperion/external_interaction/ispyb/ispyb_store.py @@ -66,7 +66,6 @@ def begin_deposition( def update_deposition( self, ispyb_ids, - data_collection_group_info: Optional[DataCollectionGroupInfo], scan_data_infos: Sequence[ScanDataInfo], ) -> IspybIds: assert ( @@ -75,9 +74,7 @@ def update_deposition( assert ( ispyb_ids.data_collection_ids ), "Attempted to store scan data without a collection" - return self._begin_or_update_deposition( - ispyb_ids, data_collection_group_info, scan_data_infos - ) + return self._begin_or_update_deposition(ispyb_ids, None, scan_data_infos) def _begin_or_update_deposition( self, diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index 071b07292..29b979f42 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -494,9 +494,7 @@ def test_ispyb_deposition_comment_correct_for_3D_on_failure( ExperimentType.GRIDSCAN_3D, ispyb_ids, ) - ispyb_ids = dummy_ispyb_3d.update_deposition( - ispyb_ids, dummy_data_collection_group_info, scan_data_infos - ) + ispyb_ids = dummy_ispyb_3d.update_deposition(ispyb_ids, scan_data_infos) dcid1 = ispyb_ids.data_collection_ids[0] # type: ignore dcid2 = ispyb_ids.data_collection_ids[1] # type: ignore dummy_ispyb_3d.end_deposition(ispyb_ids, "fail", "could not connect to devices") @@ -539,9 +537,7 @@ def test_can_store_2D_ispyb_data_correctly_when_in_error( dummy_params, dummy_scan_data_info_for_begin, experiment_type, ispyb_ids ) - ispyb_ids = ispyb.update_deposition( - ispyb_ids, dummy_data_collection_group_info, scan_data_infos - ) + ispyb_ids = ispyb.update_deposition(ispyb_ids, scan_data_infos) assert len(ispyb_ids.data_collection_ids) == exp_num_of_grids # type: ignore assert len(ispyb_ids.grid_ids) == exp_num_of_grids # type: ignore assert isinstance(ispyb_ids.data_collection_group_id, int) diff --git a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py index a5f8ad260..ad1b817b4 100644 --- a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py @@ -99,16 +99,7 @@ def test_hardware_read_events( TestData.test_descriptor_document_pre_data_collection ) callback.activity_gated_event(TestData.test_event_document_pre_data_collection) - assert_upsert_call_with( - mx.upsert_data_collection_group.mock_calls[0], - mx.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "SAD", - "sampleid": TEST_SAMPLE_ID, - }, - ) + mx.upsert_data_collection_group.assert_not_called() assert_upsert_call_with( mx.upsert_data_collection.mock_calls[0], mx.get_data_collection_params(), diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py index aeb0544e7..06946d6fd 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py @@ -134,16 +134,7 @@ def test_hardware_read_event_3d(self, mock_ispyb_conn): TestData.test_descriptor_document_pre_data_collection ) callback.activity_gated_event(TestData.test_event_document_pre_data_collection) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "Mesh3D", - "sampleid": TEST_SAMPLE_ID, - }, - ) + mx_acq.upsert_data_collection_group.assert_not_called() assert_upsert_call_with( mx_acq.upsert_data_collection.mock_calls[0], mx_acq.get_data_collection_params(), diff --git a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py b/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py index 0495ad9ba..528b22e01 100644 --- a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py +++ b/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py @@ -245,9 +245,7 @@ def test_ispyb_deposition_comment_for_3D_correct( ispyb_ids = dummy_3d_gridscan_ispyb.begin_deposition( dummy_collection_group_info, [scan_data_info_for_begin] ) - dummy_3d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, scan_data_infos_for_update - ) + dummy_3d_gridscan_ispyb.update_deposition(ispyb_ids, scan_data_infos_for_update) first_upserted_param_value_list = mock_upsert_dc.call_args_list[1][0][0] second_upserted_param_value_list = mock_upsert_dc.call_args_list[2][0][0] @@ -280,7 +278,7 @@ def test_store_3d_grid_scan( ) assert dummy_3d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, scan_data_infos_for_update + ispyb_ids, scan_data_infos_for_update ) == IspybIds( data_collection_ids=TEST_DATA_COLLECTION_IDS, data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, @@ -382,11 +380,12 @@ def test_update_deposition( mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) mx_acq.upsert_data_collection_group.assert_called_once() mx_acq.upsert_data_collection.assert_called_once() + mx_acq.upsert_data_collection_group.reset_mock() dummy_collection_group_info.sample_barcode = TEST_BARCODE actual_rows = dummy_3d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, scan_data_infos_for_update + ispyb_ids, scan_data_infos_for_update ) assert actual_rows == IspybIds( @@ -395,17 +394,7 @@ def test_update_deposition( grid_ids=TEST_GRID_INFO_IDS, ) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[1], - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "Mesh3D", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, - }, - ) + mx_acq.upsert_data_collection_group.assert_not_called() assert_upsert_call_with( mx_acq.upsert_data_collection.mock_calls[1], @@ -580,11 +569,12 @@ def test_end_deposition_happy_path( ispyb_ids = dummy_3d_gridscan_ispyb.begin_deposition( dummy_collection_group_info, [scan_data_info_for_begin] ) + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + assert len(mx_acq.upsert_data_collection_group.mock_calls) == 1 ispyb_ids = dummy_3d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, scan_data_infos_for_update + ispyb_ids, scan_data_infos_for_update ) - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - assert len(mx_acq.upsert_data_collection_group.mock_calls) == 2 + assert len(mx_acq.upsert_data_collection_group.mock_calls) == 1 assert len(mx_acq.upsert_data_collection.mock_calls) == 3 assert len(mx_acq.upsert_dc_grid.mock_calls) == 2 @@ -637,7 +627,7 @@ def test_param_keys( dummy_collection_group_info, [scan_data_info_for_begin] ) assert dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] + ispyb_ids, [scan_xy_data_info_for_update] ) == IspybIds( data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, @@ -658,9 +648,7 @@ def _test_when_grid_scan_stored_then_data_present_in_upserts( ispyb_ids = dummy_ispyb.begin_deposition( dummy_collection_group_info, [scan_data_info_for_begin] ) - dummy_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_data_info_for_update] - ) + dummy_ispyb.update_deposition(ispyb_ids, [scan_data_info_for_update]) mx_acquisition = ispyb_conn.return_value.__enter__.return_value.mx_acquisition @@ -672,7 +660,7 @@ def _test_when_grid_scan_stored_then_data_present_in_upserts( if test_group: upsert_data_collection_group_arg_list = ( - mx_acquisition.upsert_data_collection_group.call_args_list[1][0] + mx_acquisition.upsert_data_collection_group.call_args_list[0][0] ) actual = upsert_data_collection_group_arg_list[0] assert test_function(MXAcquisition.get_data_collection_group_params(), actual) @@ -747,7 +735,7 @@ def test_fail_result_run_results_in_bad_run_status( dummy_collection_group_info, [scan_data_info_for_begin] ) ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] + ispyb_ids, [scan_xy_data_info_for_update] ) dummy_2d_gridscan_ispyb.end_deposition(ispyb_ids, "fail", "test specifies failure") @@ -776,7 +764,7 @@ def test_no_exception_during_run_results_in_good_run_status( dummy_collection_group_info, [scan_data_info_for_begin] ) ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] + ispyb_ids, [scan_xy_data_info_for_update] ) dummy_2d_gridscan_ispyb.end_deposition(ispyb_ids, "success", "") diff --git a/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py b/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py index e9bf84c1e..f1cb7b0fc 100644 --- a/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py +++ b/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py @@ -326,23 +326,12 @@ def test_update_deposition( assert dummy_rotation_ispyb.update_deposition( ispyb_ids, - dummy_rotation_data_collection_group_info, [scan_data_info_for_update], ) == IspybIds( data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), ) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "SAD", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred - }, - ) + mx_acq.upsert_data_collection_group.assert_not_called() assert_upsert_call_with( mx_acq.upsert_data_collection.mock_calls[0], mx_acq.get_data_collection_params(), @@ -396,23 +385,12 @@ def test_update_deposition_with_group_id_updates( dummy_rotation_data_collection_group_info.sample_barcode = TEST_BARCODE assert dummy_rotation_ispyb.update_deposition( ispyb_ids, - dummy_rotation_data_collection_group_info, [scan_data_info_for_update], ) == IspybIds( data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), ) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "SAD", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred - }, - ) + mx_acq.upsert_data_collection_group.assert_not_called() assert_upsert_call_with( mx_acq.upsert_data_collection.mock_calls[0], mx_acq.get_data_collection_params(), @@ -462,7 +440,6 @@ def test_end_deposition_happy_path( scan_data_info_for_update.data_collection_id = ispyb_ids.data_collection_ids[0] ispyb_ids = dummy_rotation_ispyb.update_deposition( ispyb_ids, - dummy_rotation_data_collection_group_info, [scan_data_info_for_update], ) mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) @@ -535,7 +512,6 @@ def test_store_rotation_scan_uses_supplied_dcgid( assert ( store_in_ispyb.update_deposition( ispyb_ids, - dummy_rotation_data_collection_group_info, [scan_data_info_for_update], ).data_collection_group_id == dcgid From 2a4412a0223584958cc97f63091306da9798fe42 Mon Sep 17 00:00:00 2001 From: Oliver Silvester Date: Mon, 29 Apr 2024 16:23:59 +0100 Subject: [PATCH 17/18] Panda improvements --- src/hyperion/device_setup_plans/setup_panda.py | 7 +++++-- src/hyperion/experiment_plans/flyscan_xray_centre_plan.py | 5 +++++ .../experiment_plans/panda_flyscan_xray_centre_plan.py | 2 +- src/hyperion/parameters/constants.py | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/hyperion/device_setup_plans/setup_panda.py b/src/hyperion/device_setup_plans/setup_panda.py index 6d4c64918..27c44dc66 100644 --- a/src/hyperion/device_setup_plans/setup_panda.py +++ b/src/hyperion/device_setup_plans/setup_panda.py @@ -161,8 +161,6 @@ def setup_panda_for_flyscan( table = get_seq_table(parameters, exposure_distance_mm) - LOGGER.info(f"Setting PandA sequencer values: {str(table)}") - yield from bps.abs_set(panda.seq[1].table, table, group="panda-config") yield from bps.abs_set( @@ -174,6 +172,10 @@ def setup_panda_for_flyscan( # Values need to be set before blocks are enabled, so wait here yield from bps.wait(group="panda-config", timeout=GENERAL_TIMEOUT) + LOGGER.info(f"PandA sequencer table has been set to: {str(table)}") + table_readback = yield from bps.rd(panda.seq[1].table) + LOGGER.info(f"PandA sequencer table readback is: {str(table_readback)}") + yield from arm_panda_for_gridscan(panda) @@ -183,6 +185,7 @@ def arm_panda_for_gridscan(panda: HDFPanda, group="arm_panda_gridscan"): yield from bps.abs_set(panda.counter[1].enable, Enabled.ENABLED.value, group=group) # type: ignore yield from bps.abs_set(panda.pcap.arm, PcapArm.ARMED.value, group=group) # type: ignore yield from bps.wait(group=group, timeout=GENERAL_TIMEOUT) + LOGGER.info("PandA has been armed") def disarm_panda_for_gridscan(panda, group="disarm_panda_gridscan") -> MsgGenerator: diff --git a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py index 019f8e6fe..37c9e7b16 100755 --- a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py @@ -1,6 +1,7 @@ from __future__ import annotations import dataclasses +from time import time from typing import TYPE_CHECKING, Any, List, Union import bluesky.plan_stubs as bps @@ -196,12 +197,16 @@ def do_fgs(): yield from bps.wait() LOGGER.info("kicking off FGS") yield from bps.kickoff(gridscan, wait=True) + gridscan_start_time = time() LOGGER.info("Waiting for Zocalo device queue to have been cleared...") yield from bps.wait( ZOCALO_STAGE_GROUP ) # Make sure ZocaloResults queue is clear and ready to accept our new data LOGGER.info("completing FGS") yield from bps.complete(gridscan, wait=True) + LOGGER.info( + f"Gridscan motion program took {round(time()-gridscan_start_time,2)} to complete" + ) yield from do_fgs() diff --git a/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py index 0e8f421ee..c2cce1210 100755 --- a/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/panda_flyscan_xray_centre_plan.py @@ -59,7 +59,7 @@ ) PANDA_SETUP_PATH = ( - "/dls_sw/i03/software/daq_configuration/panda_configs/flyscan_base.yaml" + "/dls_sw/i03/software/daq_configuration/panda_configs/flyscan_pcap_ignore_seq.yaml" ) diff --git a/src/hyperion/parameters/constants.py b/src/hyperion/parameters/constants.py index dcd261f1e..0022e3c58 100644 --- a/src/hyperion/parameters/constants.py +++ b/src/hyperion/parameters/constants.py @@ -48,7 +48,7 @@ class DocDescriptorNames: @dataclass(frozen=True) class HardwareConstants: OAV_REFRESH_DELAY = 0.3 - PANDA_FGS_RUN_UP_DEFAULT = 0.16 + PANDA_FGS_RUN_UP_DEFAULT = 0.17 @dataclass(frozen=True) From 08a5ca8a2a38c509486d7f93c7d8a758e5163c54 Mon Sep 17 00:00:00 2001 From: Oliver Silvester Date: Tue, 30 Apr 2024 08:59:02 +0100 Subject: [PATCH 18/18] Response from review --- src/hyperion/device_setup_plans/setup_panda.py | 2 +- src/hyperion/experiment_plans/flyscan_xray_centre_plan.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hyperion/device_setup_plans/setup_panda.py b/src/hyperion/device_setup_plans/setup_panda.py index 27c44dc66..cfcb3eca9 100644 --- a/src/hyperion/device_setup_plans/setup_panda.py +++ b/src/hyperion/device_setup_plans/setup_panda.py @@ -174,7 +174,7 @@ def setup_panda_for_flyscan( LOGGER.info(f"PandA sequencer table has been set to: {str(table)}") table_readback = yield from bps.rd(panda.seq[1].table) - LOGGER.info(f"PandA sequencer table readback is: {str(table_readback)}") + LOGGER.debug(f"PandA sequencer table readback is: {str(table_readback)}") yield from arm_panda_for_gridscan(panda) diff --git a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py index 37c9e7b16..92cf99db5 100755 --- a/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py +++ b/src/hyperion/experiment_plans/flyscan_xray_centre_plan.py @@ -204,6 +204,8 @@ def do_fgs(): ) # Make sure ZocaloResults queue is clear and ready to accept our new data LOGGER.info("completing FGS") yield from bps.complete(gridscan, wait=True) + + # Remove this logging statement once metrics have been added LOGGER.info( f"Gridscan motion program took {round(time()-gridscan_start_time,2)} to complete" )