Enhance From File catalog loading to support more columns and improve Clear Table functionality

CHANGES.rst

@@ -9,6 +9,7 @@ Cubeviz
 
 Imviz
 ^^^^^
+- Enhance the Catalog Search plugin to support additional columns when loading catalog data from files. [#3359]
 
 Mosviz
 ^^^^^^

docs/imviz/plugins.rst

@@ -382,8 +382,8 @@ To load a catalog from a supported `JWST ECSV catalog file <https://jwst-pipelin
 The file must be able to be parsed by `astropy.table.Table.read` and contains the following columns:
 
 * ``'sky_centroid'``: Column with `~astropy.coordinates.SkyCoord` sky coordinates of the sources.
-* ``'label'``: Column with string identifiers of the sources. If you have numerical identifiers,
-  they will be recast as string.
+* ``'label(optional)'``: Column with string identifiers of the sources. If not provided, unique string identifiers will be generated automatically.
+  If you have numerical identifiers, they will be recast as strings.
 
 Clicking :guilabel:`SEARCH` will show markers for any entry within the filtered zoom window.
 

jdaviz/configs/imviz/plugins/catalogs/catalogs.py

@@ -46,7 +46,11 @@ class Catalogs(PluginTemplateMixin, ViewerSelectMixin, HasFileImportSelect, Tabl
             'Object ID': np.nan,
             'id': np.nan,
             'x_coord': np.nan,
-            'y_coord': np.nan}
+            'y_coord': np.nan
+            }
+
+    headers = ['Right Ascension (degrees)', 'Declination (degrees)',
+               'Object ID', 'x_coord', 'y_coord']
 
     @property
     def user_api(self):
@@ -72,11 +76,8 @@ def __init__(self, *args, **kwargs):
         self._marker_name = 'catalog_results'
 
         # initializing the headers in the table that is displayed in the UI
-        headers = ['Right Ascension (degrees)', 'Declination (degrees)',
-                   'Object ID', 'x_coord', 'y_coord']
-
-        self.table.headers_avail = headers
-        self.table.headers_visible = headers
+        self.table.headers_avail = self.headers
+        self.table.headers_visible = self.headers
         self.table._default_values_by_colname = self._default_table_values
         self.table._selected_rows_changed_callback = lambda msg: self.plot_selected_points()
         self.table.item_key = 'id'
@@ -90,13 +91,15 @@ def __init__(self, *args, **kwargs):
     def _file_parser(path):
         try:
             table = QTable.read(path)
-        except Exception:
-            return 'Could not parse file with astropy.table.QTable.read', {}
+            if not table.colnames:  # Ensure the file has columns
+                return "File contains no columns", {}
 
-        if 'sky_centroid' not in table.colnames:
-            return 'Table does not contain required sky_centroid column', {}
+            if 'sky_centroid' not in table.colnames:
+                return 'Table does not contain required sky_centroid column', {}
 
-        return '', {path: table}
+            return '', {path: table}
+        except Exception:
+            return 'Could not parse file with astropy.table.QTable.read', {}
 
     @with_spinner()
     def search(self, error_on_fail=False):
@@ -115,7 +118,7 @@ def search(self, error_on_fail=False):
 
         """
         # calling clear in the case the user forgot after searching
-        self.clear()
+        self.clear_table()
 
         # gets the current viewer
         viewer = self.viewer.selected_obj
@@ -204,6 +207,11 @@ def search(self, error_on_fail=False):
             # all exceptions when going through the UI should have prevented setting this path
             # but this exceptions might be raised here if setting from_file from the UI
             table = self.catalog.selected_obj
+            column_names = list(table.colnames)
+            self.table.headers_avail = self.headers + [
+                col for col in column_names if col not in self.headers and col not in ["sky_centroid", "label"]     # noqa:E501
+                ]
+            self.table.headers_visible = self.headers
             self.app._catalog_source_table = table
             if len(table['sky_centroid']) > self.max_sources:
                 skycoord_table = table['sky_centroid'][:self.max_sources]
@@ -271,16 +279,19 @@ def search(self, error_on_fail=False):
             if len(self.app._catalog_source_table) == 1 or self.max_sources == 1:
                 x_coordinates = [x_coordinates]
                 y_coordinates = [y_coordinates]
+            for idx, (row, x_coord, y_coord) in enumerate(zip(self.app._catalog_source_table, x_coordinates, y_coordinates)):  # noqa:E501
+                row_info = {
+                    'Right Ascension (degrees)': row['sky_centroid'].ra.deg,
+                    'Declination (degrees)': row['sky_centroid'].dec.deg,
+                    'Object ID': str(row.get('label', f"{idx + 1}")),
+                    'id': idx+1,
+                    'x_coord': x_coord,
+                    'y_coord': y_coord,
+                }
+                for col in table.colnames:
+                    if col not in ['label', 'sky_centroid']:  # Skip already processed columns
+                        row_info[col] = row[col]
 
-            for row, x_coord, y_coord in zip(self.app._catalog_source_table,
-                                             x_coordinates, y_coordinates):
-                # Check if the row contains the required keys
-                row_info = {'Right Ascension (degrees)': row['sky_centroid'].ra.deg,
-                            'Declination (degrees)': row['sky_centroid'].dec.deg,
-                            'Object ID': str(row.get('label', 'N/A')),
-                            'id': len(self.table),
-                            'x_coord': x_coord.item() if x_coord.size == 1 else x_coord,
-                            'y_coord': y_coord.item() if y_coord.size == 1 else y_coord}
                 self.table.add_item(row_info)
 
         filtered_skycoord_table = viewer.state.reference_data.coords.pixel_to_world(x_coordinates,
@@ -357,8 +368,7 @@ def zoom_to_selected(self, padding=0.02, return_bounding_box=False):
         viewer = self.app._jdaviz_helper._default_viewer
 
         selected_rows = self.table.selected_rows
-
-        if not len(selected_rows):
+        if not selected_rows:  # Check if no rows are selected
             return
 
         if padding <= 0 or padding > 1:
@@ -402,9 +412,13 @@ def vue_do_search(self, *args, **kwargs):
         # calls self.search() which handles all of the searching logic
         self.search()
 
-    def clear(self, hide_only=True):
+    def clear_table(self, hide_only=True):
         # gets the current viewer
         viewer = self.viewer.selected_obj
+        # Clear the table before performing a new search
+        self.table.items = []
+        self.table.selected_rows = []
+        self.table.selected_indices = []
 
         if not hide_only and self._marker_name in self.app.data_collection.labels:
             # resetting values
@@ -426,5 +440,5 @@ def clear(self, hide_only=True):
                 if layer_is_table_data(lyr.layer) and lyr.layer.label == self._marker_name:
                     lyr.visible = False
 
-    def vue_do_clear(self, *args, **kwargs):
-        self.clear()
+    def vue_do_clear_table(self, *args, **kwargs):
+        self.clear_table()

jdaviz/configs/imviz/plugins/catalogs/catalogs.vue

@@ -36,6 +36,16 @@
         See the <j-external-link link='https://astroquery.readthedocs.io/en/latest/gaia/gaia.html' linktext='astropy.gaia docs'></j-external-link> for details on the query defaults.
       </j-docs-link>
     </v-row>
+
+    <v-row v-if="catalog_selected && catalog_selected.endsWith('.ecsv')">
+      <v-select
+        v-model="selected_columns"
+        :items="column_names"
+        label="Select Columns"
+        multiple
+        hint="Select columns to display in the table."
+      />
+    </v-row>
 
     <v-row>
       <v-text-field
@@ -53,14 +63,6 @@
     </v-row>
 
     <v-row class="row-no-outside-padding">
-       <v-col>
-         <plugin-action-button
-            :results_isolated_to_plugin="true"
-            @click="do_clear"
-          >
-            Clear
-          </plugin-action-button>
-       </v-col>
        <v-col>
          <plugin-action-button
             :results_isolated_to_plugin="true"

jdaviz/configs/imviz/tests/test_catalogs.py

@@ -114,7 +114,7 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path):
         prev_results = catalogs_plugin.number_of_results
 
         # testing that every variable updates accordingly when markers are cleared
-        catalogs_plugin.vue_do_clear()
+        catalogs_plugin.vue_do_clear_table()
 
         assert not catalogs_plugin.results_available
 
@@ -158,14 +158,25 @@ def test_plugin_image_with_result(self, imviz_helper, tmp_path):
         assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5
         assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1488.5
 
+        # Re-populate the table with a new search
+        out_tbl = catalogs_plugin.search()
+        assert len(out_tbl) > 0
+        # Ensure at least one row is selected before zooming
+        catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]]
+        assert len(catalogs_plugin.table.selected_rows) > 0
+
         # set 'padding' to reproduce original hard-coded 50 pixel window
         # so test results don't change
         catalogs_plugin.zoom_to_selected(padding=50 / 2048)
 
-        assert imviz_helper.viewers['imviz-0']._obj.state.x_min == 858.24969
-        assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 958.38461
-        assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 278.86265
-        assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 378.8691
+        assert_allclose(
+            imviz_helper.viewers['imviz-0']._obj.state.x_min, 1022.57570000, atol=0.1)
+        assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max,
+                        1122.5757, atol=0.1)
+        assert_allclose(
+            imviz_helper.viewers['imviz-0']._obj.state.y_min, 675.29611, atol=0.1)
+        assert_allclose(
+            imviz_helper.viewers['imviz-0']._obj.state.y_max, 775.29611, atol=0.1)
 
 
 def test_from_file_parsing(imviz_helper, tmp_path):
@@ -200,7 +211,7 @@ def test_from_file_parsing(imviz_helper, tmp_path):
 def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path):
     sky = SkyCoord(ra=[337.5202807, 337.51909197, 337.51760596],
                    dec=[-20.83305528, -20.83222194, -20.83083304], unit='deg')
-    tbl = QTable({'sky_centroid': sky})
+    tbl = QTable({'sky_centroid': sky})  # Table has no "Label" column
     tbl_file = str(tmp_path / 'sky_centroid.ecsv')
     tbl.write(tbl_file, overwrite=True)
     n_entries = len(tbl)
@@ -215,6 +226,9 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path):
     out_tbl = catalogs_plugin.search(error_on_fail=True)
     assert len(out_tbl) == n_entries
     assert catalogs_plugin.number_of_results == n_entries
+    # Assert that Object ID is set to index + 1 when the label column is absent
+    for idx, item in enumerate(catalogs_plugin.table.items):
+        assert item['Object ID'] == str(idx + 1)
     assert len(imviz_helper.app.data_collection) == 2  # image + markers
 
     catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]]
@@ -240,27 +254,34 @@ def test_offline_ecsv_catalog(imviz_helper, image_2d_wcs, tmp_path):
     assert catalogs_plugin.number_of_results == n_entries
     assert len(imviz_helper.app.data_collection) == 2  # image + markers
 
-    catalogs_plugin.clear()
+    catalogs_plugin.clear_table()
 
     assert not catalogs_plugin.results_available
     assert len(imviz_helper.app.data_collection) == 2  # markers still there, just hidden
 
-    catalogs_plugin.clear(hide_only=False)
+    catalogs_plugin.clear_table(hide_only=False)
     assert not catalogs_plugin.results_available
     assert len(imviz_helper.app.data_collection) == 1  # markers gone for good
 
     assert imviz_helper.viewers['imviz-0']._obj.state.x_min == -0.5
     assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 9.5
     assert imviz_helper.viewers['imviz-0']._obj.state.y_min == -0.5
     assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 9.5
+    # Re-populate the table with a new search
+    out_tbl = catalogs_plugin.search()
+    assert len(out_tbl) > 0
+    # Ensure at least one row is selected before zooming
+    catalogs_plugin.table.selected_rows = [catalogs_plugin.table.items[0]]
+    assert len(catalogs_plugin.table.selected_rows) > 0
 
     # test the zooming using the default 'padding' of 2% of the viewer size
     # around selected points
     catalogs_plugin.zoom_to_selected()
-    assert imviz_helper.viewers['imviz-0']._obj.state.x_min == -0.19966
-    assert imviz_helper.viewers['imviz-0']._obj.state.x_max == 0.20034000000000002
-    assert imviz_helper.viewers['imviz-0']._obj.state.y_min == 0.8000100000000001
-    assert imviz_helper.viewers['imviz-0']._obj.state.y_max == 1.20001
+    assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_min, -0.19966, rtol=1e-1)
+    assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.x_max,
+                    0.20034000000000002, rtol=1e-1)
+    assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_min, 0.8000100000000001, rtol=1e-1)
+    assert_allclose(imviz_helper.viewers['imviz-0']._obj.state.y_max, 1.20001, rtol=1e-1)
 
 
 def test_zoom_to_selected(imviz_helper, image_2d_wcs, tmp_path):
@@ -334,3 +355,40 @@ def test_zoom_to_selected(imviz_helper, image_2d_wcs, tmp_path):
     # test that appropriate error is raised when padding is not a valud percentage
     with pytest.raises(ValueError, match="`padding` must be between 0 and 1."):
         catalogs_plugin.zoom_to_selected(padding=5)
+
+
+def test_offline_ecsv_catalog_with_extra_columns(imviz_helper, image_2d_wcs, tmp_path):
+    # Create a table with additional columns
+    sky = SkyCoord(ra=[337.5202807, 337.51909197, 337.51760596],
+                   dec=[-20.83305528, -20.83222194, -20.83083304], unit='deg')
+    tbl = QTable({
+        'sky_centroid': sky,
+        'flux': [1.0, 2.0, 3.0],
+        'flux_err': [0.1, 0.2, 0.3],
+        'is_extended': [False, True, False],
+        'roundness': [0.01, 0.02, 0.03],
+        'sharpness': [0.1, 0.2, 0.3]
+    })
+    tbl_file = str(tmp_path / 'extra_columns.ecsv')
+    tbl.write(tbl_file, overwrite=True)
+
+    ndd = NDData(np.ones((10, 10)), wcs=image_2d_wcs)
+    imviz_helper.load_data(ndd, data_label='data_with_wcs')
+    assert len(imviz_helper.app.data_collection) == 1
+
+    catalogs_plugin = imviz_helper.plugins['Catalog Search']._obj
+    catalogs_plugin.from_file = tbl_file
+    catalogs_plugin.catalog_selected = 'From File...'
+    catalogs_plugin.search(error_on_fail=True)
+
+    extra_columns = ['flux', 'flux_err', 'is_extended', 'roundness', 'sharpness']
+    for col in extra_columns:
+        assert col in catalogs_plugin.table.headers_avail
+
+    # Check if extra columns are populated correctly
+    for idx, item in enumerate(catalogs_plugin.table.items):
+        assert float(item['flux']) == tbl['flux'][idx]
+        assert float(item['flux_err']) == tbl['flux_err'][idx]
+        assert item['is_extended'] == tbl['is_extended'][idx]
+        assert float(item['roundness']) == tbl['roundness'][idx]
+        assert float(item['sharpness']) == tbl['sharpness'][idx]

jdaviz/core/template_mixin.py

@@ -4719,12 +4719,20 @@ def float_precision(column, item):
                 return ''
             elif isinstance(item, tuple) and np.all([np.isnan(i) for i in item]):
                 return ''
-
             elif isinstance(item, float):
                 return float_precision(column, item)
             elif isinstance(item, (list, tuple)):
                 return [float_precision(column, i) if isinstance(i, float) else i for i in item]
-
+            elif isinstance(item, (np.float32, np.float64)):
+                return float(item)
+            elif isinstance(item, u.Quantity):
+                return {"value": item.value.tolist() if item.size > 1 else item.value, "unit": str(item.unit)}     # noqa: E501
+            elif isinstance(item, np.bool_):
+                return bool(item)
+            elif isinstance(item, np.ndarray):
+                return item.tolist()
+            elif isinstance(item, tuple):
+                return tuple(json_safe(v) for v in item)
             return item
 
         if isinstance(item, QTable):
@@ -4766,6 +4774,8 @@ def clear_table(self):
         Clear all entries/markers from the current table.
         """
         self.items = []
+        self.selected_rows = []
+        self.selected_indices = []
         self._qtable = None
         self._plugin.session.hub.broadcast(PluginTableModifiedMessage(sender=self))