Improved UVData to MirParser interface

CHANGELOG.md

@@ -2,6 +2,18 @@
 All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
+### Added
+- The `catalog_name` keyword has been added to the `UVData.read` and `UVData.select`
+methods to allow users to select on source name.
+
+### Changed
+- The keywords for `UVData.read` have been changed for MIR-specific values to more
+generally match those founds with other data types.
+- The `UVData.read_mir` method has been overhauled to mak reading in of MIR data more
+efficiency in memory/processor usage.
+
+### Fixed
+- Frequency frame information for MS datasets is now correctly recorded as "TOPO".
 
 ## [2.3.1] - 2023-04-03
 

docs/uvdata_tutorial.rst

@@ -1621,7 +1621,7 @@ into a standard UVData object (which doubles its size).
   >>> print(mir_uv.polarization_array)
   [0]
   >>> print(mir_uv.data_array.shape)
-  (1, 1, 262160, 1)
+  (1, 262160, 1)
 
   >>> # Use the ``remove_flex_pol`` method to get a standard object. Note that it
   >>> # doubles the data_array size
@@ -1634,7 +1634,7 @@ into a standard UVData object (which doubles its size).
   >>> print(mir_uv.polarization_array)
   [-6 -5]
   >>> print(mir_uv.data_array.shape)
-  (1, 1, 262160, 2)
+  (1, 262160, 2)
 
 
 b) Converting between standard and flex_pol objects

pyuvdata/uvdata/mir_parser.py

@@ -309,8 +309,30 @@ def copy(self, metadata_only=False):
 
         # include all attributes, not just UVParameter ones.
         for attr in vars(self):
-            if not (metadata_only and attr in ["vis_data", "raw_data", "auto_data"]):
-                setattr(new_obj, attr, copy.deepcopy(getattr(self, attr)))
+            if issubclass(getattr(self, attr).__class__, MirMetaData):
+                setattr(new_obj, attr, getattr(self, attr).copy())
+            elif not (metadata_only and attr in ["vis_data", "raw_data", "auto_data"]):
+                if attr not in ["_metadata_attrs", "_sp_dict", "_ac_dict"]:
+                    setattr(new_obj, attr, copy.deepcopy(getattr(self, attr)))
+
+        rec_dict_list = []
+        if self._has_auto:
+            rec_dict_list.append("_ac_dict")
+        if self._has_cross:
+            rec_dict_list.append("_sp_dict")
+
+        for item in rec_dict_list:
+            new_dict = {}
+            setattr(new_obj, item, new_dict)
+            for inhid, in_dict in getattr(self, item).items():
+                new_in_dict = {}
+                new_dict[inhid] = new_in_dict
+                for key, rec_dict in in_dict.items():
+                    new_in_dict[key] = {
+                        "start_idx": rec_dict["start_idx"],
+                        "end_idx": rec_dict["end_idx"],
+                        "chan_avg": rec_dict["chan_avg"],
+                    }
 
         for item in self._metadata_attrs:
             new_obj._metadata_attrs[item] = getattr(new_obj, item)
@@ -394,8 +416,15 @@ def _fix_int_dict(self, data_type):
             (auto-correlations).
         """
         for ifile, idict in self._file_dict.items():
-            if not idict[data_type]["ignore_header"]:
-                int_dict = copy.deepcopy(idict[data_type]["int_dict"])
+            # Don't attempt to fix kludged headers, since this implies that the file
+            # metadata cannot be trusted
+            if idict[data_type]["ignore_header"]:
+                warnings.warn(
+                    "Cannot fix %s file headers for %s, skipping." % (data_type, ifile)
+                )
+                continue
+
+            int_dict = copy.deepcopy(idict[data_type]["int_dict"])
 
             # Each file's inhid is allowed to be different than the objects inhid --
             # this is used in cases when combining multiple files together (via
@@ -562,6 +591,8 @@ def _read_packdata(file_dict, inhid_arr, data_type="cross", use_mmap=False):
             if not indv_file_dict[data_type]["ignore_header"]:
                 good_check = True
                 for inhid, idict in int_data_dict.items():
+                    if inhid not in int_dict:
+                        continue
                     # There is very little to check in the packdata records, so make
                     # sure that this entry corresponds to the inhid and size we expect.
                     good_check &= idict["inhid"] == int_dict[inhid]["inhid"]
@@ -1377,7 +1408,7 @@ def load_data(
         # If we are potentially downselecting data (typically used when calling select),
         # make sure that we actually have all the data we need loaded.
         if allow_downselect or (allow_downselect is None):
-            if load_cross:
+            if load_cross and not (self.vis_data is None and self.raw_data is None):
                 try:
                     self._downselect_data(
                         select_vis=load_vis, select_raw=load_raw, select_auto=False
@@ -1387,7 +1418,7 @@ def load_data(
                     if allow_downselect:
                         warnings.warn("Cannot downselect cross-correlation data.")
 
-            if load_auto:
+            if load_auto and self.auto_data is not None:
                 try:
                     self._downselect_data(
                         select_vis=False, select_raw=False, select_auto=True
@@ -1444,11 +1475,21 @@ def unload_data(self, unload_vis=True, unload_raw=True, unload_auto=True):
             Default is True.
         """
         if unload_vis:
-            self.vis_data = None
+            if self.vis_data is not None:
+                for item in self.vis_data.values():
+                    del item["data"]
+                    del item["flags"]
+                self.vis_data = None
             self._tsys_applied = False
-        if unload_raw:
+        if unload_raw and self.raw_data is not None:
+            for item in self.raw_data.values():
+                del item["data"]
+                del item["scale_fac"]
             self.raw_data = None
-        if unload_auto:
+        if unload_auto and self.auto_data is not None:
+            for item in self.auto_data.values():
+                del item["data"]
+                del item["flags"]
             self.auto_data = None
 
     def _update_filter(self, update_data=None):
@@ -1473,69 +1514,70 @@ def _update_filter(self, update_data=None):
         # Start by cascading the filters up -- from largest metadata tables to the
         # smallest. First up, spec win -> baseline
         if not np.all(self.sp_data.get_mask()):
-            mask_update |= self.bl_data.set_mask(header_key=set(self.sp_data["blhid"]))
+            mask_update |= self.bl_data._make_key_mask(self.sp_data)
 
         # Now do baseline -> antennas. Special handling required because of the
         # lack of a unique index key for this table.
-        if not (np.all(self.bl_data.get_mask()) and not self._has_auto):
-            key_list = set(
-                self.bl_data.get_value(["iant1", "inhid"], return_tuples=True)
-                + self.bl_data.get_value(["iant2", "inhid"], return_tuples=True)
+        if self._has_auto or not np.all(self.bl_data.get_mask()):
+            mask = self.eng_data._make_key_mask(
+                self.bl_data,
+                check_field=("iant1", "inhid"),
+                set_mask=False,
+                use_cipher=True,
+            )
+            mask |= self.eng_data._make_key_mask(
+                self.bl_data,
+                check_field=("iant2", "inhid"),
+                set_mask=False,
+                use_cipher=True,
             )
 
             if self._has_auto:
-                key_list.union(
-                    self.ac_data.get_value(["antenna", "inhid"], return_tuples=True)
+                mask |= self.eng_data._make_key_mask(
+                    self.ac_data, set_mask=False, use_cipher=True
                 )
 
-            mask_update |= self.eng_data.set_mask(header_key=key_list)
+            mask_update |= self.eng_data.set_mask(mask=mask)
 
         # Now antennas -> int
         if not np.all(self.eng_data.get_mask()):
-            mask_update |= self.in_data.set_mask(header_key=set(self.eng_data["inhid"]))
+            mask_update |= self.in_data._make_key_mask(self.eng_data)
 
         # And weather scan -> int
         if not np.all(self.we_data.get_mask()):
-            mask_update |= self.in_data.set_mask(
-                where=("ints", "eq", self.we_data["ints"])
-            )
+            mask_update |= self.in_data._make_key_mask(self.we_data, reverse=True)
 
         # We now cascade the masks downward. First up, int -> weather scan
-        mask_update |= self.we_data.set_mask(header_key=self.in_data["ints"])
+        mask_update |= self.we_data._make_key_mask(self.in_data)
+
+        # Next, do int -> baseline
+        mask_update |= self.bl_data._make_key_mask(self.in_data, reverse=True)
 
         # Next, ant -> baseline. Again this requires a little extra special
         # handling, since eng_data doesn't have a unique header key.
-        bl_eng_mask = np.logical_and(
-            self.eng_data.get_mask(
-                header_key=self.bl_data.get_value(
-                    ["iant1", "inhid"], return_tuples=True, use_mask=False
-                )
-            ),
-            self.eng_data.get_mask(
-                header_key=self.bl_data.get_value(
-                    ["iant2", "inhid"], return_tuples=True, use_mask=False
-                )
-            ),
+        mask = self.bl_data._make_key_mask(
+            self.eng_data,
+            check_field=("iant1", "inhid"),
+            set_mask=False,
+            use_cipher=True,
+            reverse=True,
         )
-        mask_update |= self.bl_data.set_mask(mask=bl_eng_mask)
-
-        # Next, do int -> baseline
-        mask_update |= self.bl_data.set_mask(
-            where=("inhid", "eq", self.in_data["inhid"])
+        mask &= self.bl_data._make_key_mask(
+            self.eng_data,
+            check_field=("iant2", "inhid"),
+            set_mask=False,
+            use_cipher=True,
+            reverse=True,
         )
+        mask_update |= self.bl_data.set_mask(mask=mask)
 
         # Finally, do baseline -> spec win for the crosses...
-        mask_update |= self.sp_data.set_mask(
-            where=("blhid", "eq", self.bl_data["blhid"])
-        )
+        mask_update |= self.sp_data._make_key_mask(self.bl_data, reverse=True)
+
         # ...and the autos.
         if self._has_auto:
-            mask_update |= self.ac_data.set_mask(
-                mask=self.eng_data.get_mask(
-                    header_key=self.ac_data.get_value(
-                        ["antenna", "inhid"], return_tuples=True, use_mask=False
-                    )
-                )
+            mask_update |= self.ac_data._make_key_mask(
+                self.eng_data, use_cipher=True, reverse=True
             )
 
         if update_data or (update_data is None):
@@ -1615,10 +1657,10 @@ def _fix_acdata(self):
         sel_mask = np.isin(self.ac_data._data["iband"], unique_bands)
 
         self.ac_data._data = self.ac_data._data[sel_mask]
-        self.ac_data._mask = np.ones(len(self.ac_data), dtype=bool)
+        self.ac_data._mask = np.ones(self.ac_data._size, dtype=bool)
 
         # Set up the header index for the object, and then construct the header key dict
-        self.ac_data._data["achid"] = np.arange(1, len(self.ac_data) + 1)
+        self.ac_data._data["achid"] = np.arange(1, self.ac_data._size + 1)
         self.ac_data._set_header_key_index_dict()
 
         # Now that we've got vitals, we want to import in metadata from some of the
@@ -2610,7 +2652,7 @@ def select(
 
         for attr, mask in search_dict.items():
             self._metadata_attrs[attr].set_mask(
-                mask=mask, reset=reset, and_mask=and_mask
+                mask=mask, reset=reset, and_mask=and_mask, use_mask=False
             )
 
         # Now that we've screened the data that we want, update the object appropriately

pyuvdata/uvdata/ms.py

@@ -720,7 +720,7 @@ def _write_ms_spectralwindow(self, filepath):
             sw_table.putcell("RESOLUTION", idx, ch_width[ch_mask])
             # TODO: These are placeholders for now, but should be replaced with
             # actual frequency reference info (once UVData handles that)
-            sw_table.putcell("MEAS_FREQ_REF", idx, VEL_DICT["LSRK"])
+            sw_table.putcell("MEAS_FREQ_REF", idx, VEL_DICT["TOPO"])
             sw_table.putcell("REF_FREQUENCY", idx, freq_array[0])
 
         sw_table.done()
@@ -1414,10 +1414,15 @@ def _parse_pyuvdata_frame_ref(self, frame_name, epoch_val, raise_error=True):
 
         ref_name = None
         try:
-            ref_name = reverse_dict[(str(frame_name), float(epoch_val))]
+            ref_name = reverse_dict[
+                (str(frame_name), 2000.0 if (epoch_val is None) else float(epoch_val))
+            ]
         except KeyError as err:
+            epoch_msg = (
+                "no epoch" if epoch_val is None else f"epoch {format(epoch_val,'g')}"
+            )
             message = (
-                f"Frame {frame_name} (epoch {format(epoch_val,'g')}) does not have a "
+                f"Frame {frame_name} ({epoch_msg}) does not have a "
                 "corresponding match to supported frames in the MS file format."
             )
             if raise_error: