From 129272c17e8e0c19b19a5f5728399e9e1aebbf38 Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Wed, 20 Nov 2024 17:59:39 -0700 Subject: [PATCH 1/9] In remote_group, improve error message when there is a timeout getting a CSC summary state. --- python/lsst/ts/observatory/control/remote_group.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/lsst/ts/observatory/control/remote_group.py b/python/lsst/ts/observatory/control/remote_group.py index 680c3cc3..868bc260 100644 --- a/python/lsst/ts/observatory/control/remote_group.py +++ b/python/lsst/ts/observatory/control/remote_group.py @@ -443,11 +443,16 @@ async def get_state( timeout=self.fast_timeout ) return salobj.State(ss.summaryState) - except asyncio.TimeoutError as e: + except asyncio.TimeoutError: if ignore_timeout: return None else: - raise e + raise asyncio.TimeoutError( + f"Timeout getting summary state for {component}. " + "This usually means that no historical data is available. " + "You might have to recycle the CSC state manually to recover " + "from this condition." + ) async def next_state(self, component: str) -> salobj.State: """Get summary state for component. From 1ad5d7b4fd1bf6f86546835267d4ff66fccf96a5 Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Fri, 22 Nov 2024 11:32:49 -0700 Subject: [PATCH 2/9] In maintel/mtcs, update _handle_m1m3_hardpoint_correction_command to wait for m1m3_settle_time after enabling/disabling force balance. --- python/lsst/ts/observatory/control/maintel/mtcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lsst/ts/observatory/control/maintel/mtcs.py b/python/lsst/ts/observatory/control/maintel/mtcs.py index 98054079..54fe39f7 100644 --- a/python/lsst/ts/observatory/control/maintel/mtcs.py +++ b/python/lsst/ts/observatory/control/maintel/mtcs.py @@ -1547,9 +1547,9 @@ async def _handle_m1m3_hardpoint_correction_command( ) except (asyncio.TimeoutError, salobj.base.AckTimeoutError): self.log.warning("Command timed out, continuing.") - else: - self.log.debug("Waiting for force balance system to settle.") + self.log.info("Waiting for force balance system to settle.") await self._wait_force_balance_system_state(enable=enable) + await asyncio.sleep(self.m1m3_settle_time) else: self.log.warning( f"Hardpoint corrections already in desired state ({enable=}). Nothing to do." From e021e44e5f6d96e8f87962dce114fc322b8b7f24 Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Wed, 27 Nov 2024 11:38:35 -0700 Subject: [PATCH 3/9] In maintel/mtcs, reduce m1m3 setting time. --- python/lsst/ts/observatory/control/maintel/mtcs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/ts/observatory/control/maintel/mtcs.py b/python/lsst/ts/observatory/control/maintel/mtcs.py index 54fe39f7..ebd4ed7c 100644 --- a/python/lsst/ts/observatory/control/maintel/mtcs.py +++ b/python/lsst/ts/observatory/control/maintel/mtcs.py @@ -155,7 +155,7 @@ def __init__( # timeout to raise m1m3, in seconds. self.m1m3_raise_timeout = 600.0 # time it takes for m1m3 to settle after a slew finishes. - self.m1m3_settle_time = 5.0 + self.m1m3_settle_time = 0.0 # Tolerance on the stability of the balance force magnitude self.m1m3_force_magnitude_stable_tolerance = 50.0 From 25362c3bab53e458e839dba5825f4afada4c3f18 Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Thu, 28 Nov 2024 11:39:46 -0700 Subject: [PATCH 4/9] In base_tcs, update _handle_in_position method to expose the timeout to use when handling the initial state race condition. --- python/lsst/ts/observatory/control/base_tcs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/lsst/ts/observatory/control/base_tcs.py b/python/lsst/ts/observatory/control/base_tcs.py index 3b8067a8..fdf96dd6 100644 --- a/python/lsst/ts/observatory/control/base_tcs.py +++ b/python/lsst/ts/observatory/control/base_tcs.py @@ -1903,6 +1903,7 @@ async def _handle_in_position( timeout: float, settle_time: float = 5.0, component_name: str = "", + race_condition_timeout: float = 5.0, ) -> str: """Handle inPosition event. @@ -1917,6 +1918,8 @@ async def _handle_in_position( component_name : `str`, optional The name of the component. This is used in log messages and to construct a return message when in position. + race_condition_timeout : `float` + Timeout to use when handling race condition (in seconds). Returns ------- @@ -1931,7 +1934,7 @@ async def _handle_in_position( in_position = await in_position_event.aget(timeout=self.fast_timeout) self.log.debug(f"{component_name} in position: {in_position.inPosition}.") - _settle_time = max([settle_time, self.fast_timeout]) + _settle_time = max([settle_time, race_condition_timeout]) if in_position.inPosition: self.log.debug( From f0e81f9dabc18047616cf1931f496ffcce44384b Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Thu, 28 Nov 2024 11:40:53 -0700 Subject: [PATCH 5/9] In maintel/mtcs, use the custom race_condition_timeout for checking the mount and hexapod are in position. --- python/lsst/ts/observatory/control/maintel/mtcs.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/lsst/ts/observatory/control/maintel/mtcs.py b/python/lsst/ts/observatory/control/maintel/mtcs.py index ebd4ed7c..d63f5b6a 100644 --- a/python/lsst/ts/observatory/control/maintel/mtcs.py +++ b/python/lsst/ts/observatory/control/maintel/mtcs.py @@ -128,6 +128,16 @@ def __init__( self.open_dome_shutter_time = 1200.0 self.timeout_hardpoint_test_status = 600.0 + # There is a race condition when commanding the mount + # to a new target, which is the time it takes for it + # to start moving after we send the command to the pointing. + # This timeout expresses how long to wait when handling this + # race condition. + self.mtmount_race_condition_timeout = 3.0 + # Similar to the mtmount_race_condition_timeout, this is + # used to check the in position event race condition for + # the hexapod when checking if it is ready to take data. + self.hexapod_ready_to_take_data_timeout = 0.5 self.tel_park_el = 80.0 self.tel_park_az = 0.0 @@ -503,12 +513,14 @@ async def wait_for_mtmount_inposition( timeout=timeout, settle_time=0.0, component_name="MTMount elevation", + race_condition_timeout=self.mtmount_race_condition_timeout, ), self._handle_in_position( self.rem.mtmount.evt_azimuthInPosition, timeout=timeout, settle_time=0.0, component_name="MTMount azimuth", + race_condition_timeout=self.mtmount_race_condition_timeout, ), ) @@ -2595,6 +2607,7 @@ async def _ready_to_take_data(self) -> None: timeout=self.long_timeout, settle_time=0.0, component_name="Camera Hexapod", + race_condition_timeout=self.hexapod_ready_to_take_data_timeout, ), ) except asyncio.TimeoutError: From 74b3802b377afb2788644f7f144137bcf018c9f3 Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Fri, 29 Nov 2024 14:18:11 -0700 Subject: [PATCH 6/9] In maintel/mtcs.py, update wait_for_rotator_inposition to use a lower race condition timeout and to not await any settling time. --- python/lsst/ts/observatory/control/maintel/mtcs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/lsst/ts/observatory/control/maintel/mtcs.py b/python/lsst/ts/observatory/control/maintel/mtcs.py index d63f5b6a..89badac1 100644 --- a/python/lsst/ts/observatory/control/maintel/mtcs.py +++ b/python/lsst/ts/observatory/control/maintel/mtcs.py @@ -138,6 +138,10 @@ def __init__( # used to check the in position event race condition for # the hexapod when checking if it is ready to take data. self.hexapod_ready_to_take_data_timeout = 0.5 + # Similar to the mtmount_race_condition_timeout, this is + # used to check the in position event race condition for + # the rotator when checking if it is in position. + self.mtrotator_race_condition_timeout = 3.0 self.tel_park_el = 80.0 self.tel_park_az = 0.0 @@ -584,8 +588,9 @@ async def wait_for_rotator_inposition( return await self._handle_in_position( self.rem.mtrotator.evt_inPosition, timeout=timeout, - settle_time=self.tel_settle_time, + settle_time=0.0, component_name="MTRotator", + race_condition_timeout=self.mtrotator_race_condition_timeout, ) async def dome_az_in_position(self) -> str: From c0a61c45386cdeee039e067839c784b1c50cb7b4 Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Mon, 2 Dec 2024 11:43:20 -0700 Subject: [PATCH 7/9] In maintel/mtcs, remove old backward compatibility with m1m3 FATables not being in ts-xml. --- python/lsst/ts/observatory/control/maintel/mtcs.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/lsst/ts/observatory/control/maintel/mtcs.py b/python/lsst/ts/observatory/control/maintel/mtcs.py index 89badac1..accdd297 100644 --- a/python/lsst/ts/observatory/control/maintel/mtcs.py +++ b/python/lsst/ts/observatory/control/maintel/mtcs.py @@ -33,11 +33,7 @@ from lsst.ts import salobj, utils from lsst.ts.utils import angle_diff from lsst.ts.xml.enums import MTM1M3, MTM2, MTDome, MTMount, MTPtg, MTRotator - -try: - from lsst.ts.xml.tables.m1m3 import FATable -except ImportError: - from lsst.ts.criopy.M1M3FATable import FATABLE as FATable +from lsst.ts.xml.tables.m1m3 import FATable from ..base_tcs import BaseTCS from ..constants import mtcs_constants From 77aebfff32464e727ad90f1c768a01303bccb2ea Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Mon, 2 Dec 2024 11:45:43 -0700 Subject: [PATCH 8/9] Remove ts_criopy from the list of ups dependencies. --- ups/ts_observatory_control.table | 1 - 1 file changed, 1 deletion(-) diff --git a/ups/ts_observatory_control.table b/ups/ts_observatory_control.table index 8afe3132..839a67d9 100644 --- a/ups/ts_observatory_control.table +++ b/ups/ts_observatory_control.table @@ -6,7 +6,6 @@ setupRequired(sconsUtils) setupRequired(ts_utils) setupRequired(ts_salobj) setupRequired(ts_idl) -setupRequired(ts_criopy) envPrepend(PATH, ${PRODUCT_DIR}/bin) From 9d2bf5d232eff32a0514081794c3c0b95a78228d Mon Sep 17 00:00:00 2001 From: edennihy Date: Sun, 1 Dec 2024 23:06:09 -0700 Subject: [PATCH 9/9] Add news fragments. --- doc/news/DM-47641.feature.1.rst | 1 + doc/news/DM-47641.feature.2.rst | 1 + doc/news/DM-47641.feature.3.rst | 1 + doc/news/DM-47641.feature.4.rst | 1 + doc/news/DM-47641.feature.rst | 1 + doc/news/DM-47641.removal.rst | 1 + 6 files changed, 6 insertions(+) create mode 100644 doc/news/DM-47641.feature.1.rst create mode 100644 doc/news/DM-47641.feature.2.rst create mode 100644 doc/news/DM-47641.feature.3.rst create mode 100644 doc/news/DM-47641.feature.4.rst create mode 100644 doc/news/DM-47641.feature.rst create mode 100644 doc/news/DM-47641.removal.rst diff --git a/doc/news/DM-47641.feature.1.rst b/doc/news/DM-47641.feature.1.rst new file mode 100644 index 00000000..e95f86ac --- /dev/null +++ b/doc/news/DM-47641.feature.1.rst @@ -0,0 +1 @@ +In maintel/mtcs, reduce m1m3 setting time. diff --git a/doc/news/DM-47641.feature.2.rst b/doc/news/DM-47641.feature.2.rst new file mode 100644 index 00000000..85a88e39 --- /dev/null +++ b/doc/news/DM-47641.feature.2.rst @@ -0,0 +1 @@ +In base_tcs, update _handle_in_position method to expose the timeout to use when handling the initial state race condition. 81efa99 Tiago Ribeiro Dec 1, 2024 at 10:59 PM diff --git a/doc/news/DM-47641.feature.3.rst b/doc/news/DM-47641.feature.3.rst new file mode 100644 index 00000000..3a24fc84 --- /dev/null +++ b/doc/news/DM-47641.feature.3.rst @@ -0,0 +1 @@ +In maintel/mtcs, use the custom race_condition_timeout for checking the mount and hexapod are in position. diff --git a/doc/news/DM-47641.feature.4.rst b/doc/news/DM-47641.feature.4.rst new file mode 100644 index 00000000..1360bf5f --- /dev/null +++ b/doc/news/DM-47641.feature.4.rst @@ -0,0 +1 @@ +In maintel/mtcs.py, update wait_for_rotator_inposition to use a lower race condition timeout and to not await any settling time. diff --git a/doc/news/DM-47641.feature.rst b/doc/news/DM-47641.feature.rst new file mode 100644 index 00000000..4105194f --- /dev/null +++ b/doc/news/DM-47641.feature.rst @@ -0,0 +1 @@ +In maintel/mtcs, update _handle_m1m3_hardpoint_correction_command to wait for m1m3_settle_time after enabling/disabling force balance. diff --git a/doc/news/DM-47641.removal.rst b/doc/news/DM-47641.removal.rst new file mode 100644 index 00000000..69951a0b --- /dev/null +++ b/doc/news/DM-47641.removal.rst @@ -0,0 +1 @@ +Removed backwards compatibility with m1m3 FATables not being in ts-xml. \ No newline at end of file