From ae5ffc40f684c67efe03e194a6d110b2a85ba0b4 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Wed, 25 Oct 2023 09:19:29 -0700 Subject: [PATCH 1/4] Ensure block parsing does not fail with minimal data --- python/lsst/summit/utils/blockUtils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/lsst/summit/utils/blockUtils.py b/python/lsst/summit/utils/blockUtils.py index a7d75616..02f3b369 100644 --- a/python/lsst/summit/utils/blockUtils.py +++ b/python/lsst/summit/utils/blockUtils.py @@ -236,6 +236,15 @@ def augmentData(self): but is also much harder to maintain/debug, as the vectorized regexes are hard to work with, and to know which row is causing problems. """ + if 'lastCheckpoint' not in self.data.columns: + self.log.warning("No lastCheckpoint column in data, can't parse block data") + # add the columns that would have been added for consistency + self.data['blockNum'] = pd.Series() + self.data['blockId'] = pd.Series() + self.data['blockDayObs'] = pd.Series() + self.data['blockSeqNum'] = pd.Series() + return + data = self.data blockPattern = r"BLOCK-(\d+)" blockIdPattern = r"(BL\d+(?:_\w+)+)" From 0773232673d9020d598c14766d04ede3017ab511 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Wed, 25 Oct 2023 09:20:02 -0700 Subject: [PATCH 2/4] Add extra safety when adding blockInfo --- python/lsst/summit/utils/tmaUtils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/lsst/summit/utils/tmaUtils.py b/python/lsst/summit/utils/tmaUtils.py index f05ef531..742d2e08 100644 --- a/python/lsst/summit/utils/tmaUtils.py +++ b/python/lsst/summit/utils/tmaUtils.py @@ -1318,7 +1318,15 @@ def addBlockDataToEvents(self, dayObs, events): `list` of `lsst.summit.utils.tmaUtils.TMAEvent` One or more events to get the block data for. """ - blockParser = BlockParser(dayObs, client=self.client) + try: + blockParser = BlockParser(dayObs, client=self.client) + except Exception as e: + # adding the block data should never cause a failure so if we can't + # get the block data, log a warning and return. It is, however, + # never expected, so use log.exception to get the full traceback + # and scare users so it gets reported + self.log.exception(f'Failed to parse block data for {dayObs=}, {e}') + return blocks = blockParser.getBlockNums() blockDict = {} for block in blocks: From 7fb4a21ebb3c71957885f5682a4603472668204b Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Thu, 26 Oct 2023 05:59:34 -0700 Subject: [PATCH 3/4] Improve warning message when failing to parse --- python/lsst/summit/utils/blockUtils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/lsst/summit/utils/blockUtils.py b/python/lsst/summit/utils/blockUtils.py index 02f3b369..4f9286b0 100644 --- a/python/lsst/summit/utils/blockUtils.py +++ b/python/lsst/summit/utils/blockUtils.py @@ -237,7 +237,9 @@ def augmentData(self): are hard to work with, and to know which row is causing problems. """ if 'lastCheckpoint' not in self.data.columns: - self.log.warning("No lastCheckpoint column in data, can't parse block data") + nRows = len(self.data) + self.log.warning(f"Found {nRows} rows of data and no 'lastCheckpoint' column was in the data," + " so block data cannot be parsed.") # add the columns that would have been added for consistency self.data['blockNum'] = pd.Series() self.data['blockId'] = pd.Series() From 88c23928efe8b4bea7126075f6edcbfd51f37967 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Thu, 26 Oct 2023 06:06:40 -0700 Subject: [PATCH 4/4] Add warning for lack of lastCheckpoint in slow augment too --- python/lsst/summit/utils/blockUtils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/lsst/summit/utils/blockUtils.py b/python/lsst/summit/utils/blockUtils.py index 4f9286b0..207eb4f8 100644 --- a/python/lsst/summit/utils/blockUtils.py +++ b/python/lsst/summit/utils/blockUtils.py @@ -212,6 +212,11 @@ def augmentDataSlow(self): data['blockDayObs'] = pd.Series() data['blockSeqNum'] = pd.Series() + if 'lastCheckpoint' not in self.data.columns: + nRows = len(self.data) + self.log.warning(f"Found {nRows} rows of data and no 'lastCheckpoint' column was in the data," + " so block data cannot be parsed.") + for index, row in data.iterrows(): rowStr = row['lastCheckpoint']