From 2f28144dcb087ca285115928e74ec31bf4874be1 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Sat, 14 Dec 2024 23:09:38 -0500 Subject: [PATCH 01/22] Add functionality to load catalogs from a file --- .../imviz/plugins/catalogs/catalogs.py | 60 ++++++++++++------- .../imviz/plugins/catalogs/catalogs.vue | 10 ++++ jdaviz/core/template_mixin.py | 11 +++- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py index 13ec7beda8..2de55dfbaf 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py @@ -45,7 +45,11 @@ class Catalogs(PluginTemplateMixin, ViewerSelectMixin, HasFileImportSelect, Tabl 'Object ID': np.nan, 'id': np.nan, 'x_coord': np.nan, - 'y_coord': np.nan} + 'y_coord': np.nan + } + + headers = ['Right Ascension (degrees)', 'Declination (degrees)', + 'Object ID', 'x_coord', 'y_coord'] @property def user_api(self): @@ -70,11 +74,8 @@ def __init__(self, *args, **kwargs): self._marker_name = 'catalog_results' # initializing the headers in the table that is displayed in the UI - headers = ['Right Ascension (degrees)', 'Declination (degrees)', - 'Object ID', 'x_coord', 'y_coord'] - - self.table.headers_avail = headers - self.table.headers_visible = headers + self.table.headers_avail = self.headers + self.table.headers_visible = self.headers self.table._default_values_by_colname = self._default_table_values self.table._selected_rows_changed_callback = lambda msg: self.plot_selected_points() self.table.item_key = 'id' @@ -88,13 +89,15 @@ def __init__(self, *args, **kwargs): def _file_parser(path): try: table = QTable.read(path) - except Exception: - return 'Could not parse file with astropy.table.QTable.read', {} + if not table.colnames: # Ensure the file has columns + return "File contains no columns", {} - if 'sky_centroid' not in table.colnames: - return 'Table does not contain required sky_centroid column', {} + if 'sky_centroid' not in table.colnames: + return 'Table does not contain required sky_centroid column', {} - return '', {path: table} + return '', {path: table} + except Exception: + return 'Could not parse file with astropy.table.QTable.read', {} @with_spinner() def search(self, error_on_fail=False): @@ -202,6 +205,11 @@ def search(self, error_on_fail=False): # all exceptions when going through the UI should have prevented setting this path # but this exceptions might be raised here if setting from_file from the UI table = self.catalog.selected_obj + column_names = list(table.colnames) + self.table.headers_avail = self.headers + [ + col for col in column_names if col not in self.headers + ] + self.table.headers_visible = self.headers self.app._catalog_source_table = table if len(table['sky_centroid']) > self.max_sources: skycoord_table = table['sky_centroid'][:self.max_sources] @@ -269,16 +277,28 @@ def search(self, error_on_fail=False): if len(self.app._catalog_source_table) == 1 or self.max_sources == 1: x_coordinates = [x_coordinates] y_coordinates = [y_coordinates] + for idx, (row, x_coord, y_coord) in enumerate(zip(self.app._catalog_source_table, x_coordinates, y_coordinates)): + sky_centroid = row['sky_centroid'] + if isinstance(sky_centroid, SkyCoord): + ra_deg = sky_centroid.ra.deg + dec_deg = sky_centroid.dec.deg + else: + # Handle scalar or list-like case + ra_deg = sky_centroid.ra[idx].deg + dec_deg = sky_centroid.dec[idx].deg + + row_info = { + 'Right Ascension (degrees)': ra_deg, + 'Declination (degrees)': dec_deg, + 'Object ID': str(row.get('label', 'N/A')), + 'id': idx, + 'x_coord': x_coord, + 'y_coord': y_coord, + } + for col in table.colnames: + if col not in ['label', 'sky_centroid']: # Skip already processed columns + row_info[col] = row[col] - for row, x_coord, y_coord in zip(self.app._catalog_source_table, - x_coordinates, y_coordinates): - # Check if the row contains the required keys - row_info = {'Right Ascension (degrees)': row['sky_centroid'].ra.deg, - 'Declination (degrees)': row['sky_centroid'].dec.deg, - 'Object ID': str(row.get('label', 'N/A')), - 'id': len(self.table), - 'x_coord': x_coord.item() if x_coord.size == 1 else x_coord, - 'y_coord': y_coord.item() if y_coord.size == 1 else y_coord} self.table.add_item(row_info) filtered_skycoord_table = viewer.state.reference_data.coords.pixel_to_world(x_coordinates, diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue b/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue index 830b02672d..3ba693d8e1 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue @@ -36,6 +36,16 @@ See the for details on the query defaults. + + + + 1 else item.value + elif isinstance(item, np.bool_): + return bool(item) return item if isinstance(item, QTable): @@ -4707,6 +4711,9 @@ def float_precision(column, item): # Row does not have .items() implemented item = {k: v for k, v in zip(item.keys(), item.values())} + # Clean and JSON-safe processing + item = {k: json_safe(k, v) for k, v in item.items()} + # save original sent values to the cached QTable object if self._qtable is None: self._qtable = QTable([item]) From 746d8db169978a4f754072c6c4ee1c21b4aad135 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Sun, 15 Dec 2024 12:06:12 -0500 Subject: [PATCH 02/22] Fix table and viewer clearing functionality --- .../configs/imviz/plugins/catalogs/catalogs.py | 17 +++++------------ jdaviz/core/template_mixin.py | 13 ++++++++----- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py index 2de55dfbaf..0b4c6e2551 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py @@ -207,7 +207,7 @@ def search(self, error_on_fail=False): table = self.catalog.selected_obj column_names = list(table.colnames) self.table.headers_avail = self.headers + [ - col for col in column_names if col not in self.headers + col for col in column_names if col not in self.headers and col not in ["sky_centroid", "label"] ] self.table.headers_visible = self.headers self.app._catalog_source_table = table @@ -278,18 +278,9 @@ def search(self, error_on_fail=False): x_coordinates = [x_coordinates] y_coordinates = [y_coordinates] for idx, (row, x_coord, y_coord) in enumerate(zip(self.app._catalog_source_table, x_coordinates, y_coordinates)): - sky_centroid = row['sky_centroid'] - if isinstance(sky_centroid, SkyCoord): - ra_deg = sky_centroid.ra.deg - dec_deg = sky_centroid.dec.deg - else: - # Handle scalar or list-like case - ra_deg = sky_centroid.ra[idx].deg - dec_deg = sky_centroid.dec[idx].deg - row_info = { - 'Right Ascension (degrees)': ra_deg, - 'Declination (degrees)': dec_deg, + 'Right Ascension (degrees)': row['sky_centroid'].ra.deg, + 'Declination (degrees)': row['sky_centroid'].dec.deg, 'Object ID': str(row.get('label', 'N/A')), 'id': idx, 'x_coord': x_coord, @@ -384,6 +375,8 @@ def vue_do_search(self, *args, **kwargs): def clear(self, hide_only=True): # gets the current viewer viewer = self.viewer.selected_obj + # Clear the table before performing a new search + self.table.items = [] if not hide_only and self._marker_name in self.app.data_collection.labels: # resetting values diff --git a/jdaviz/core/template_mixin.py b/jdaviz/core/template_mixin.py index 28b6944225..720d72e08d 100644 --- a/jdaviz/core/template_mixin.py +++ b/jdaviz/core/template_mixin.py @@ -4695,12 +4695,16 @@ def float_precision(column, item): return float_precision(column, item) elif isinstance(item, (list, tuple)): return [float_precision(column, i) if isinstance(i, float) else i for i in item] - elif isinstance(item, np.ndarray): - return item.tolist() # Convert arrays to lists + elif isinstance(item, (np.float32, np.float64)): + return float(item) elif isinstance(item, u.Quantity): return item.value.tolist() if item.size > 1 else item.value elif isinstance(item, np.bool_): return bool(item) + elif isinstance(item, np.ndarray): + return item.tolist() + elif isinstance(item, tuple): + return tuple(json_safe(v) for v in item) return item if isinstance(item, QTable): @@ -4711,9 +4715,6 @@ def float_precision(column, item): # Row does not have .items() implemented item = {k: v for k, v in zip(item.keys(), item.values())} - # Clean and JSON-safe processing - item = {k: json_safe(k, v) for k, v in item.items()} - # save original sent values to the cached QTable object if self._qtable is None: self._qtable = QTable([item]) @@ -4745,6 +4746,8 @@ def clear_table(self): Clear all entries/markers from the current table. """ self.items = [] + self.selected_rows = [] + self.selected_indices = [] self._qtable = None self._plugin.session.hub.broadcast(PluginTableModifiedMessage(sender=self)) From 485846e7912acaf559e4b1e36b56ad2a4040ab69 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Sun, 15 Dec 2024 22:20:32 -0500 Subject: [PATCH 03/22] Remove redundant 'Clear' button and combine its functionality with 'Clear Table' for intuitive behavior --- jdaviz/configs/imviz/plugins/catalogs/catalogs.py | 10 ++++++---- jdaviz/configs/imviz/plugins/catalogs/catalogs.vue | 8 -------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py index 0b4c6e2551..7c92853672 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py @@ -116,7 +116,7 @@ def search(self, error_on_fail=False): """ # calling clear in the case the user forgot after searching - self.clear() + self.clear_table() # gets the current viewer viewer = self.viewer.selected_obj @@ -372,11 +372,13 @@ def vue_do_search(self, *args, **kwargs): # calls self.search() which handles all of the searching logic self.search() - def clear(self, hide_only=True): + def clear_table(self, hide_only=True): # gets the current viewer viewer = self.viewer.selected_obj # Clear the table before performing a new search self.table.items = [] + self.table.selected_rows = [] + self.table.selected_indices = [] if not hide_only and self._marker_name in self.app.data_collection.labels: # resetting values @@ -398,5 +400,5 @@ def clear(self, hide_only=True): if layer_is_table_data(lyr.layer) and lyr.layer.label == self._marker_name: lyr.visible = False - def vue_do_clear(self, *args, **kwargs): - self.clear() + def vue_do_clear_table(self, *args, **kwargs): + self.clear_table() diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue b/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue index 3ba693d8e1..b9475faeff 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.vue @@ -63,14 +63,6 @@ - - - Clear - - Date: Mon, 16 Dec 2024 22:34:15 -0500 Subject: [PATCH 04/22] Codestyle --- jdaviz/configs/imviz/plugins/catalogs/catalogs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py index 7c92853672..817546ed41 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py @@ -207,7 +207,7 @@ def search(self, error_on_fail=False): table = self.catalog.selected_obj column_names = list(table.colnames) self.table.headers_avail = self.headers + [ - col for col in column_names if col not in self.headers and col not in ["sky_centroid", "label"] + col for col in column_names if col not in self.headers and col not in ["sky_centroid", "label"] # noqa:E501 ] self.table.headers_visible = self.headers self.app._catalog_source_table = table @@ -277,7 +277,7 @@ def search(self, error_on_fail=False): if len(self.app._catalog_source_table) == 1 or self.max_sources == 1: x_coordinates = [x_coordinates] y_coordinates = [y_coordinates] - for idx, (row, x_coord, y_coord) in enumerate(zip(self.app._catalog_source_table, x_coordinates, y_coordinates)): + for idx, (row, x_coord, y_coord) in enumerate(zip(self.app._catalog_source_table, x_coordinates, y_coordinates)): # noqa:E501 row_info = { 'Right Ascension (degrees)': row['sky_centroid'].ra.deg, 'Declination (degrees)': row['sky_centroid'].dec.deg, @@ -337,7 +337,8 @@ def plot_selected_points(self): def vue_zoom_in(self, *args, **kwargs): """This function will zoom into the image based on the selected points""" selected_rows = self.table.selected_rows - + if not selected_rows: # Check if no rows are selected + return x = [float(coord['x_coord']) for coord in selected_rows] y = [float(coord['y_coord']) for coord in selected_rows] From 73aad209e62fc84f25e36ce5abbb9465d76e60e8 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Mon, 16 Dec 2024 22:38:49 -0500 Subject: [PATCH 05/22] Add change log --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 81ab9d30ce..85dfbae36a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,6 +21,8 @@ Imviz - Catalog Search plugin now exposes a maximum sources limit for all catalogs and resolves an edge case when loading a catalog from a file that only contains one source. [#3337] +- Catalog Search plugin now supports loading catalog data directly from files. [#3359] + Mosviz ^^^^^^ From d19de556991806657fb65f1b2c5edf7aa812b7a4 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Mon, 16 Dec 2024 22:45:53 -0500 Subject: [PATCH 06/22] Update tests --- jdaviz/configs/imviz/tests/test_catalogs.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 0e6e960ddc..2e9aca11e5 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -113,7 +113,7 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path): prev_results = catalogs_plugin.number_of_results # testing that every variable updates accordingly when markers are cleared - catalogs_plugin.vue_do_clear() + catalogs_plugin.vue_do_clear_table() assert not catalogs_plugin.results_available @@ -237,12 +237,12 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path): assert catalogs_plugin.number_of_results == n_entries assert len(imviz_helper.app.data_collection) == 2 # image + markers - catalogs_plugin.clear() + catalogs_plugin.clear_table() assert not catalogs_plugin.results_available assert len(imviz_helper.app.data_collection) == 2 # markers still there, just hidden - catalogs_plugin.clear(hide_only=False) + catalogs_plugin.clear_table(hide_only=False) assert not catalogs_plugin.results_available assert len(imviz_helper.app.data_collection) == 1 # markers gone for good @@ -250,7 +250,14 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path): assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 9.5 assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5 assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 9.5 + # Re-populate the table with a new search + out_tbl = catalogs_plugin.search() + assert len(out_tbl) > 0 + # Ensure at least one row is selected before zooming + catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]] + assert len(catalogs_plugin.table.selected_rows) > 0 + # Now zoom in catalogs_plugin.vue_zoom_in() assert imviz_helper.viewers['imviz-0']._obj.state.x_min == -49.99966 From 5ece13aa4c0c7c71329a49b75cd753355460a1a6 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Wed, 18 Dec 2024 11:25:39 -0500 Subject: [PATCH 07/22] Fix failing test --- jdaviz/configs/imviz/tests/test_catalogs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 2e9aca11e5..2ba11aa2c5 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -28,6 +28,7 @@ from astropy.nddata import NDData from astropy.coordinates import SkyCoord from astropy.table import Table, QTable +from numpy.testing import assert_allclose @pytest.mark.remote_data @@ -260,7 +261,7 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path): # Now zoom in catalogs_plugin.vue_zoom_in() - assert imviz_helper.viewers['imviz-0']._obj.state.x_min == -49.99966 - assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 50.00034 - assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -48.99999 - assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 51.00001 + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_min, -49.99966, rtol=1e-6) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max, 50.00034, rtol=1e-6) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_min, -48.99999, rtol=1e-6) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_max, 51.00001, rtol=1e-6) From 3fb6df9b4f2ba1b1b88571e4fa1b6035106f97b4 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Wed, 18 Dec 2024 15:01:14 -0500 Subject: [PATCH 08/22] Fix the test, ensuring to first select a row before zooming in. --- jdaviz/configs/imviz/tests/test_catalogs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 2ba11aa2c5..a51d88857c 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -158,12 +158,16 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path): assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5 assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1488.5 + # First select a row + catalogs_plugin.table.selected_rows = [ + catalogs_plugin.table.items[0]] + # Then zoom in to the selected catalogs_plugin.vue_zoom_in() - assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 858.24969 - assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 958.38461 - assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 278.86265 - assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 378.8691 + assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 1022.5631800000001 + assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 1122.56318 + assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 675.29611 + assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 775.29611 def test_from_file_parsing(imviz_helper, tmp_path): From 640b61569c1ecca0cd99e843331cfbcacd47b7e3 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Thu, 19 Dec 2024 11:11:48 -0500 Subject: [PATCH 09/22] Update change log --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 85dfbae36a..3d0c4f1237 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,7 +21,7 @@ Imviz - Catalog Search plugin now exposes a maximum sources limit for all catalogs and resolves an edge case when loading a catalog from a file that only contains one source. [#3337] -- Catalog Search plugin now supports loading catalog data directly from files. [#3359] +- Enhance the Catalog Search plugin to support additional columns when loading catalog data from files. [#3359] Mosviz ^^^^^^ From d7d74393d0a2fba906fd67f5b6876ce7ddd754d9 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Thu, 19 Dec 2024 12:48:10 -0500 Subject: [PATCH 10/22] Preserve the unit when serializing --- jdaviz/core/template_mixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jdaviz/core/template_mixin.py b/jdaviz/core/template_mixin.py index 720d72e08d..cead280d57 100644 --- a/jdaviz/core/template_mixin.py +++ b/jdaviz/core/template_mixin.py @@ -4698,7 +4698,7 @@ def float_precision(column, item): elif isinstance(item, (np.float32, np.float64)): return float(item) elif isinstance(item, u.Quantity): - return item.value.tolist() if item.size > 1 else item.value + return {"value": item.value.tolist() if item.size > 1 else item.value, "unit": str(item.unit)} elif isinstance(item, np.bool_): return bool(item) elif isinstance(item, np.ndarray): From 7799594bb1e3d4f9187d3dedf661956370d9fc61 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 20 Dec 2024 10:59:46 -0500 Subject: [PATCH 11/22] Generate default Object IDs when label column is missing --- jdaviz/configs/imviz/plugins/catalogs/catalogs.py | 4 ++-- jdaviz/core/template_mixin.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py index 817546ed41..9e0fe50bc5 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py @@ -281,8 +281,8 @@ def search(self, error_on_fail=False): row_info = { 'Right Ascension (degrees)': row['sky_centroid'].ra.deg, 'Declination (degrees)': row['sky_centroid'].dec.deg, - 'Object ID': str(row.get('label', 'N/A')), - 'id': idx, + 'Object ID': str(row.get('label', f"{idx + 1}")), + 'id': idx+1, 'x_coord': x_coord, 'y_coord': y_coord, } diff --git a/jdaviz/core/template_mixin.py b/jdaviz/core/template_mixin.py index cead280d57..e7e4825732 100644 --- a/jdaviz/core/template_mixin.py +++ b/jdaviz/core/template_mixin.py @@ -4698,7 +4698,7 @@ def float_precision(column, item): elif isinstance(item, (np.float32, np.float64)): return float(item) elif isinstance(item, u.Quantity): - return {"value": item.value.tolist() if item.size > 1 else item.value, "unit": str(item.unit)} + return {"value": item.value.tolist() if item.size > 1 else item.value, "unit": str(item.unit)} # noqa: E501 elif isinstance(item, np.bool_): return bool(item) elif isinstance(item, np.ndarray): From a96babd559e45ae36369831390882363cbc1a1ea Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 20 Dec 2024 11:15:56 -0500 Subject: [PATCH 12/22] Update docs related to label column in catalogs --- docs/imviz/plugins.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/imviz/plugins.rst b/docs/imviz/plugins.rst index 57801499f6..c501687f5b 100644 --- a/docs/imviz/plugins.rst +++ b/docs/imviz/plugins.rst @@ -382,8 +382,8 @@ To load a catalog from a supported `JWST ECSV catalog file Date: Mon, 23 Dec 2024 13:02:03 -0500 Subject: [PATCH 13/22] Update tests after merge --- jdaviz/configs/imviz/tests/test_catalogs.py | 30 +++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 6c9e1e69a5..9d46440d72 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -159,14 +159,21 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path): assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5 assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1488.5 + # Re-populate the table with a new search + out_tbl = catalogs_plugin.search() + assert len(out_tbl) > 0 + # Ensure at least one row is selected before zooming + catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]] + assert len(catalogs_plugin.table.selected_rows) > 0 + # set 'padding' to reproduce original hard-coded 50 pixel window # so test results don't change catalogs_plugin.zoom_to_selected(padding=50 / 2048) - assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 1022.5631800000001 - assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 1122.56318 - assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 675.29611 - assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 775.29611 + assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 858.24969 + assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 958.38461 + assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 278.86265 + assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 378.8691 def test_from_file_parsing(imviz_helper, tmp_path): @@ -254,14 +261,21 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path): assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 9.5 assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5 assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 9.5 + # Re-populate the table with a new search + out_tbl = catalogs_plugin.search() + assert len(out_tbl) > 0 + # Ensure at least one row is selected before zooming + catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]] + assert len(catalogs_plugin.table.selected_rows) > 0 # test the zooming using the default 'padding' of 2% of the viewer size # around selected points catalogs_plugin.zoom_to_selected() - assert imviz_helper.viewers['imviz-0']._obj.state.x_min == -0.19966 - assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 0.20034000000000002 - assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 0.8000100000000001 - assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1.20001 + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_min, -0.19966, rtol=1e-1) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max, + 0.20034000000000002, rtol=1e-1) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_min, 0.8000100000000001, rtol=1e-1) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_max, 1.20001, rtol=1e-1) def test_zoom_to_selected(imviz_helper, image_2d_wcs, tmp_path): From 670cf319dfc24557dec4a13c40791c4ebc4479e0 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Mon, 23 Dec 2024 13:34:22 -0500 Subject: [PATCH 14/22] Update test --- jdaviz/configs/imviz/tests/test_catalogs.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 9d46440d72..3cb104b18e 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -29,7 +29,6 @@ from astropy.io import fits from astropy.nddata import NDData from astropy.table import Table, QTable -from numpy.testing import assert_allclose @pytest.mark.remote_data @@ -170,10 +169,10 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path): # so test results don't change catalogs_plugin.zoom_to_selected(padding=50 / 2048) - assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 858.24969 - assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 958.38461 - assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 278.86265 - assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 378.8691 + assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 1022.5757000000001 + assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 1122.5757 + assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 704.7727144165947 + assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 745.8271655834053 def test_from_file_parsing(imviz_helper, tmp_path): From 0b22f24edfcd72372670166ab24212e3d2db95a4 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Mon, 23 Dec 2024 13:44:42 -0500 Subject: [PATCH 15/22] Move changelog to correct milestone --- CHANGES.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6f9afc1d70..5f62b66061 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,7 @@ Cubeviz Imviz ^^^^^ +- Enhance the Catalog Search plugin to support additional columns when loading catalog data from files. [#3359] Mosviz ^^^^^^ @@ -108,8 +109,6 @@ Imviz - Catalog Search plugin now exposes a maximum sources limit for all catalogs and resolves an edge case when loading a catalog from a file that only contains one source. [#3337] -- Enhance the Catalog Search plugin to support additional columns when loading catalog data from files. [#3359] - - Catalog Search plugin ``zoom_to_selected`` is now in the public API. The default zoom level changed from a fixed 50 pixels to a zoom window that is a fraction of the image size (default 2%) to address and issue with zooming when using a small From 04bf9993129997426084772246bfc4697691d713 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Wed, 25 Dec 2024 23:15:47 -0500 Subject: [PATCH 16/22] Adjust test assertions --- jdaviz/configs/imviz/tests/test_catalogs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 3cb104b18e..70c6353d08 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -169,10 +169,14 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path): # so test results don't change catalogs_plugin.zoom_to_selected(padding=50 / 2048) - assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 1022.5757000000001 - assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 1122.5757 - assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 704.7727144165947 - assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 745.8271655834053 + assert_allclose( + imviz_helper.viewers['imviz-0']._obj.state.x_min, 1022.57570000, atol=0.1) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max, + 1122.5757, atol=0.1) + assert_allclose( + imviz_helper.viewers['imviz-0']._obj.state.y_min, 675.29611, atol=0.1) + assert_allclose( + imviz_helper.viewers['imviz-0']._obj.state.y_max, 775.29611, atol=0.1) def test_from_file_parsing(imviz_helper, tmp_path): From b1f2277366926e6e2813bfda60f5e1fa369e337a Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Wed, 25 Dec 2024 23:45:16 -0500 Subject: [PATCH 17/22] Add test for loading catalogs with additional columns --- jdaviz/configs/imviz/tests/test_catalogs.py | 42 ++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 70c6353d08..b439297cb3 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -211,7 +211,7 @@ def test_from_file_parsing(imviz_helper, tmp_path): def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path): sky = SkyCoord(ra=[337.5202807, 337.51909197, 337.51760596], dec=[-20.83305528, -20.83222194, -20.83083304], unit='deg') - tbl = QTable({'sky_centroid': sky}) + tbl = QTable({'sky_centroid': sky}) # Table has no "Label" column tbl_file = str(tmp_path / 'sky_centroid.ecsv') tbl.write(tbl_file, overwrite=True) n_entries = len(tbl) @@ -226,6 +226,9 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path): out_tbl = catalogs_plugin.search(error_on_fail=True) assert len(out_tbl) == n_entries assert catalogs_plugin.number_of_results == n_entries + # Assert that Object ID is set to index + 1 when the label column is absent + for idx, item in enumerate(catalogs_plugin.table.items): + assert item['Object ID'] == str(idx + 1) assert len(imviz_helper.app.data_collection) == 2 # image + markers catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]] @@ -352,3 +355,40 @@ def test_zoom_to_selected(imviz_helper, image_2d_wcs, tmp_path): # test that appropriate error is raised when padding is not a valud percentage with pytest.raises(ValueError, match="`padding` must be between 0 and 1."): catalogs_plugin.zoom_to_selected(padding=5) + + +def test_offline_ecsv_catalog_with_extra_columns(imviz_helper, image_2d_wcs, tmp_path): + # Create a table with additional columns + sky = SkyCoord(ra=[337.5202807, 337.51909197, 337.51760596], + dec=[-20.83305528, -20.83222194, -20.83083304], unit='deg') + tbl = QTable({ + 'sky_centroid': sky, + 'flux': [1.0, 2.0, 3.0], + 'flux_err': [0.1, 0.2, 0.3], + 'is_extended': [False, True, False], + 'roundness': [0.01, 0.02, 0.03], + 'sharpness': [0.1, 0.2, 0.3] + }) + tbl_file = str(tmp_path / 'extra_columns.ecsv') + tbl.write(tbl_file, overwrite=True) + + ndd = NDData(np.ones((10, 10)), wcs=image_2d_wcs) + imviz_helper.load_data(ndd, data_label='data_with_wcs') + assert len(imviz_helper.app.data_collection) == 1 + + catalogs_plugin = imviz_helper.plugins['Catalog Search']._obj + catalogs_plugin.from_file = tbl_file + catalogs_plugin.catalog_selected = 'From File...' + catalogs_plugin.search(error_on_fail=True) + + extra_columns = ['flux', 'flux_err', 'is_extended', 'roundness', 'sharpness'] + for col in extra_columns: + assert col in catalogs_plugin.table.headers_avail + + # Check if extra columns are populated correctly + for idx, item in enumerate(catalogs_plugin.table.items): + assert float(item['flux']) == tbl['flux'][idx] + assert float(item['flux_err']) == tbl['flux_err'][idx] + assert item['is_extended'] == tbl['is_extended'][idx] + assert float(item['roundness']) == tbl['roundness'][idx] + assert float(item['sharpness']) == tbl['sharpness'][idx] From b8ae09e5c66d8c983f092527691b0dc0e9500b50 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Thu, 2 Jan 2025 12:33:28 -0500 Subject: [PATCH 18/22] Update change log --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 5f62b66061..7eb527130d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,8 @@ Cubeviz Imviz ^^^^^ - Enhance the Catalog Search plugin to support additional columns when loading catalog data from files. [#3359] + +- Catalog Search ``clear_table`` now removes all associated markers from the viewer. [#3359] Mosviz ^^^^^^ From dcb597195e87f7eb63c4179145a7aa25d211a1ed Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Thu, 2 Jan 2025 12:35:26 -0500 Subject: [PATCH 19/22] Update based on feedback --- .../imviz/plugins/catalogs/catalogs.py | 27 ++++++++++--------- jdaviz/configs/imviz/tests/test_catalogs.py | 4 +-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py index a20a02caf9..0fdbafbdbf 100644 --- a/jdaviz/configs/imviz/plugins/catalogs/catalogs.py +++ b/jdaviz/configs/imviz/plugins/catalogs/catalogs.py @@ -91,16 +91,17 @@ def __init__(self, *args, **kwargs): def _file_parser(path): try: table = QTable.read(path) - if not table.colnames: # Ensure the file has columns - return "File contains no columns", {} - - if 'sky_centroid' not in table.colnames: - return 'Table does not contain required sky_centroid column', {} - - return '', {path: table} except Exception: return 'Could not parse file with astropy.table.QTable.read', {} + if not table.colnames: # Ensure the file has columns + return "File contains no columns", {} + + if 'sky_centroid' not in table.colnames: + return 'Table does not contain required sky_centroid column', {} + + return '', {path: table} + @with_spinner() def search(self, error_on_fail=False): """Search the catalog, display markers on the viewer, and return results if available. @@ -207,10 +208,9 @@ def search(self, error_on_fail=False): # all exceptions when going through the UI should have prevented setting this path # but this exceptions might be raised here if setting from_file from the UI table = self.catalog.selected_obj - column_names = list(table.colnames) + column_names = table.colnames self.table.headers_avail = self.headers + [ - col for col in column_names if col not in self.headers and col not in ["sky_centroid", "label"] # noqa:E501 - ] + col for col in column_names if col not in self.headers] self.table.headers_visible = self.headers self.app._catalog_source_table = table if len(table['sky_centroid']) > self.max_sources: @@ -284,12 +284,15 @@ def search(self, error_on_fail=False): 'Right Ascension (degrees)': row['sky_centroid'].ra.deg, 'Declination (degrees)': row['sky_centroid'].dec.deg, 'Object ID': str(row.get('label', f"{idx + 1}")), - 'id': idx+1, + 'id': len(self.table), 'x_coord': x_coord, 'y_coord': y_coord, } + # Add sky_centroid and label explicitly to row_info + row_info['sky_centroid'] = row['sky_centroid'] + row_info['label'] = row.get('label', f"{idx + 1}") for col in table.colnames: - if col not in ['label', 'sky_centroid']: # Skip already processed columns + if col not in self.headers: # Skip already processed columns row_info[col] = row[col] self.table.add_item(row_info) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index b439297cb3..2b1b3dd8a4 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -159,8 +159,8 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path): assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1488.5 # Re-populate the table with a new search - out_tbl = catalogs_plugin.search() - assert len(out_tbl) > 0 + with pytest.warns(ResourceWarning): + catalogs_plugin.search(error_on_fail=True) # Ensure at least one row is selected before zooming catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]] assert len(catalogs_plugin.table.selected_rows) > 0 From d692f8676fd652a68eb94e37dc89fbf3714dd28d Mon Sep 17 00:00:00 2001 From: Hatice Karatay <66814693+haticekaratay@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:42:50 -0500 Subject: [PATCH 20/22] Apply suggestions from code review Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com> --- docs/imviz/plugins.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/imviz/plugins.rst b/docs/imviz/plugins.rst index c501687f5b..2d9d87c0c2 100644 --- a/docs/imviz/plugins.rst +++ b/docs/imviz/plugins.rst @@ -382,7 +382,8 @@ To load a catalog from a supported `JWST ECSV catalog file Date: Thu, 2 Jan 2025 15:32:13 -0500 Subject: [PATCH 21/22] Update CHANGES.rst Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com> --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index edd9bd00de..f12d30111d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,7 @@ Cubeviz Imviz ^^^^^ + - Enhance the Catalog Search plugin to support additional columns when loading catalog data from files. [#3359] - Catalog Search ``clear_table`` now removes all associated markers from the viewer. [#3359] From 974bee4daa9a575985e995b441a05860a388b765 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 3 Jan 2025 11:08:23 -0500 Subject: [PATCH 22/22] Adjust test tolerance --- jdaviz/configs/imviz/tests/test_catalogs.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jdaviz/configs/imviz/tests/test_catalogs.py b/jdaviz/configs/imviz/tests/test_catalogs.py index 2b1b3dd8a4..956c32af68 100644 --- a/jdaviz/configs/imviz/tests/test_catalogs.py +++ b/jdaviz/configs/imviz/tests/test_catalogs.py @@ -171,8 +171,8 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path): assert_allclose( imviz_helper.viewers['imviz-0']._obj.state.x_min, 1022.57570000, atol=0.1) - assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max, - 1122.5757, atol=0.1) + assert_allclose( + imviz_helper.viewers['imviz-0']._obj.state.x_max, 1122.5757, atol=0.1) assert_allclose( imviz_helper.viewers['imviz-0']._obj.state.y_min, 675.29611, atol=0.1) assert_allclose( @@ -277,11 +277,11 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path): # test the zooming using the default 'padding' of 2% of the viewer size # around selected points catalogs_plugin.zoom_to_selected() - assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_min, -0.19966, rtol=1e-1) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_min, -0.19966, rtol=1e-4) assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max, - 0.20034000000000002, rtol=1e-1) - assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_min, 0.8000100000000001, rtol=1e-1) - assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_max, 1.20001, rtol=1e-1) + 0.20034000000000002, rtol=1e-4) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_min, 0.8000100000000001, rtol=1e-4) + assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_max, 1.20001, rtol=1e-4) def test_zoom_to_selected(imviz_helper, image_2d_wcs, tmp_path):