Select main lobe 🔭 🛰️

hera_cal/chunker.py

@@ -68,7 +68,7 @@ def chunk_files(filenames, inputfile, outputfile, chunk_size, type="data",
                 polarizations = chunked_files.pols
         if spw_range is None:
             spw_range = (0, chunked_files.Nfreqs)
-        data, flags, nsamples = chunked_files.read(axis='blt', polarizations=polarizations,
+        data, flags, nsamples = chunked_files.read(polarizations=polarizations,
                                                    freq_chans=range(spw_range[0], spw_range[1]))
     elif type == 'gains':
         chunked_files.read()

hera_cal/delay_filter.py

@@ -145,9 +145,9 @@ def load_delay_filter_and_write(datafile_list, baseline_list=None, calfile_list=
     else:
         if baseline_list is None:
             if len(hd.filepaths) > 1:
-                baseline_list = list(hd.bls.values())[0]
+                baseline_list = list(hd.antpairs.values())[0]
             else:
-                baseline_list = hd.bls
+                baseline_list = hd.antpairs
         if spw_range is None:
             spw_range = [0, hd.Nfreqs]
         freqs = hd.freq_array.flatten()[spw_range[0]:spw_range[1]]
@@ -169,7 +169,8 @@ def load_delay_filter_and_write(datafile_list, baseline_list=None, calfile_list=
             Nbls_per_load = len(baseline_list)
         for i in range(0, len(baseline_list), Nbls_per_load):
             df = DelayFilter(hd, input_cal=cals)
-            df.read(bls=baseline_list[i:i + Nbls_per_load], frequencies=freqs)
+            df.read(bls=baseline_list[i:i + Nbls_per_load],
+                    frequencies=freqs, polarizations=polarizations)
             if avg_red_bllens:
                 df.avg_red_baseline_vectors()
             if external_flags is not None:

hera_cal/tests/test_frf.py

@@ -15,9 +15,8 @@
 import unittest
 from scipy import stats
 from scipy import constants
-from pyuvdata import UVFlag
-
-
+from pyuvdata import UVFlag, UVBeam
+from .. import utils
 from .. import datacontainer, io, frf
 from ..data import DATA_PATH
 
@@ -264,7 +263,13 @@ def test_time_average_argparser_multifile(self):
         assert args.flag_output is None
         assert args.filetype == "uvh5"
 
-    def test_tophat_frfilter(self):
+    @pytest.mark.parametrize(
+        "center_frates, avg_red_bllens, pass_data",
+        [(True, True, False), (True, False, False), (False, True, False), (False, False, False),
+         (True, True, True), (True, False, True),
+         (False, True, True), (False, False, True)],
+    )
+    def test_tophat_frfilter(self, center_frates, avg_red_bllens, pass_data):
         fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
         k = (24, 25, 'ee')
         frfil = frf.FRFilter(fname, filetype='miriad')
@@ -282,18 +287,22 @@ def test_tophat_frfilter(self):
         fname = os.path.join(DATA_PATH, "zen.2458043.12552.xx.HH.uvORA")
         k = (24, 25, 'ee')
         # check successful run when avg_red_bllens is True and when False.
-        for avg_red_bllens in [True, False]:
-            frfil = frf.FRFilter(fname, filetype='miriad')
-            frfil.read(bls=[k])
-            if avg_red_bllens:
-                frfil.avg_red_baseline_vectors()
-            wgts = {k: np.ones_like(frfil.flags[k], dtype=np.float)}
-            wgts[k][:, 0] = 0.0
-            frfil.tophat_frfilter(keys=[k], wgts=wgts, tol=1e-5, window='blackman-harris', skip_wgt=0.1, maxiter=100)
-            assert frfil.clean_info[k][(0, frfil.Nfreqs)]['status']['axis_0'][0] == 'skipped'
-            np.testing.assert_array_equal(frfil.clean_flags[k][:, 0], np.ones_like(frfil.flags[k][:, 0]))
-            np.testing.assert_array_equal(frfil.clean_model[k][:, 0], np.zeros_like(frfil.clean_resid[k][:, 0]))
-            np.testing.assert_array_equal(frfil.clean_resid[k][:, 0], np.zeros_like(frfil.clean_resid[k][:, 0]))
+        frfil = frf.FRFilter(fname, filetype='miriad')
+        frfil.read(bls=[k])
+        if avg_red_bllens:
+            frfil.avg_red_baseline_vectors()
+        wgts = {k: np.ones_like(frfil.flags[k], dtype=np.float)}
+        wgts[k][:, 0] = 0.0
+        if pass_data:
+            data_kwargs = {'data': frfil.data}
+        else:
+            data_kwargs = {}
+        frfil.tophat_frfilter(keys=[k], wgts=wgts, tol=1e-5, window='blackman-harris', skip_wgt=0.1,
+                              maxiter=100, center_before_filtering=center_frates, **data_kwargs)
+        assert frfil.clean_info[k][(0, frfil.Nfreqs)]['status']['axis_0'][0] == 'skipped'
+        np.testing.assert_array_equal(frfil.clean_flags[k][:, 0], np.ones_like(frfil.flags[k][:, 0]))
+        np.testing.assert_array_equal(frfil.clean_model[k][:, 0], np.zeros_like(frfil.clean_resid[k][:, 0]))
+        np.testing.assert_array_equal(frfil.clean_resid[k][:, 0], np.zeros_like(frfil.clean_resid[k][:, 0]))
 
     def test_load_tophat_frfilter_and_write_baseline_list(self, tmpdir):
         tmp_path = tmpdir.strpath
@@ -423,6 +432,95 @@ def test_load_tophat_frfilter_and_write_multifile(self, tmpdir):
         for k in doutput:
             assert doutput[k].shape == d[k].shape
 
+    def test_build_fringe_rate_profiles(self):
+        test_beam = os.path.join(DATA_PATH, "efield_test_nside16.beamfits")
+        test_data = os.path.join(DATA_PATH, "fr_unittest_data_ds.uvh5")
+        uvd = UVData()
+        uvd.read_uvh5(test_data)
+        uvb = UVBeam()
+        uvb.read_beamfits(test_beam)
+        fr_grid, profiles = frf.build_fringe_rate_profiles(uvd, uvb)
+        assert len(fr_grid) == uvd.Ntimes
+
+    def test_get_fringe_rate_limits(self):
+        # simulations constructed with the notebook at https://drive.google.com/file/d/1jPPSmL3nqQbp7tTgP77j9KC0802iWyow/view?usp=sharing
+        test_beam = os.path.join(DATA_PATH, "fr_unittest_beam.beamfits")
+        test_data = os.path.join(DATA_PATH, "fr_unittest_data_ds.uvh5")
+        uvd = UVData()
+        uvd.read_uvh5(test_data)
+        myfrf = frf.FRFilter(uvd)
+        sim_c_frates = {}
+        sim_w_frates = {}
+        uvb = UVBeam()
+        uvb.read_beamfits(test_beam)
+        c_frs, w_frs = frf.get_fringe_rate_limits(uvd, uvb, percentile_low=10, percentile_high=90)
+        for bl in c_frs:
+            # fft data
+            myfrf.fft_data(data=myfrf.data, ax='time', keys=[bl], overwrite=True, window='bh')
+            csum = np.cumsum(np.abs(myfrf.dfft[bl]) ** 2.)
+            csum /= csum.max()
+            csum = np.hstack([[0], csum, [1.]])
+            dfr = np.median(np.diff(myfrf.frates))
+            frates = np.hstack([[myfrf.frates.min() - dfr], myfrf.frates, [myfrf.frates.max() + dfr]])
+            frlow, frhigh = (frates[np.argmin(np.abs(csum - 0.1))], frates[np.argmin(np.abs(csum - 0.9))])
+            sim_c_frates[bl] = .5 * (frlow + frhigh)
+            sim_w_frates[bl] = .5 * np.abs(frlow - frhigh)
+            sim_w_frates[utils.reverse_bl(bl)] = sim_w_frates[bl]
+            sim_c_frates[utils.reverse_bl(bl)] = -sim_c_frates[bl]
+
+        for bl in sim_c_frates:
+            assert np.isclose(c_frs[bl], sim_c_frates[bl], atol=0.3, rtol=0.)
+            assert np.isclose(w_frs[bl], sim_w_frates[bl], atol=0.3, rtol=0.)
+        # test providing fringe-rate profiles.
+        fr_grid, profiles = frf.build_fringe_rate_profiles(uvd, uvb)
+        c_frs, w_frs = frf.get_fringe_rate_limits(uvd, uvb, frate_profiles=profiles)
+        for bl in sim_c_frates:
+            assert np.isclose(c_frs[bl], sim_c_frates[bl], atol=0.3, rtol=0.)
+            assert np.isclose(w_frs[bl], sim_w_frates[bl], atol=0.3, rtol=0.)
+        assert pytest.raises(ValueError, frf.get_fringe_rate_limits, uvd, None, None)
+
+    @pytest.mark.parametrize(
+        "pre_filter",
+        [True, False],
+    )
+    def test_load_tophat_frfilter_and_write_beam_frates(self, tmpdir, pre_filter):
+        # simulations constructed with the notebook at https://drive.google.com/file/d/1jPPSmL3nqQbp7tTgP77j9KC0802iWyow/view?usp=sharing
+        # load in primary beam model and isotropic noise model of sky.
+        test_beam = os.path.join(DATA_PATH, "fr_unittest_beam.beamfits")
+        test_data = os.path.join(DATA_PATH, "fr_unittest_data_ds.uvh5")
+        tmp_path = tmpdir.strpath
+        resid_outfilename = os.path.join(tmp_path, 'resid.uvh5')
+        CLEAN_outfilename = os.path.join(tmp_path, 'model.uvh5')
+        filled_outfilename = os.path.join(tmp_path, 'filled.uvh5')
+        # perform cleaning.
+        frf.load_tophat_frfilter_and_write(datafile_list=[test_data], uvbeam=test_beam, mode='dpss_leastsq', filled_outfilename=filled_outfilename,
+                                           CLEAN_outfilename=CLEAN_outfilename, frate_standoff=0.075, pre_filter_modes_between_lobe_minimum_and_zero=pre_filter,
+                                           res_outfilename=resid_outfilename, percentile_high=97.5, percentile_low=2.5)
+        hd_input = io.HERAData(test_data)
+        data, flags, nsamples = hd_input.read()
+        hd_resid = io.HERAData(resid_outfilename)
+        data_r, flags_r, nsamples_r = hd_resid.read()
+        hd_filled = io.HERAData(filled_outfilename)
+        data_f, flags_f, nsamples_f = hd_filled.read()
+
+        for bl in data:
+            assert np.mean(np.abs(data_r[bl]) ** 2.) <= .2 * np.mean(np.abs(data[bl]) ** 2.)
+
+    def test_sky_frates_minfrate_and_to_filter(self):
+        # test edge frates
+        V = frf.FRFilter(os.path.join(DATA_PATH, "PyGSM_Jy_downselect.uvh5"))
+        V.read()
+        for to_filter in [None, list(V.data.keys())[:1]]:
+            cfrates, wfrates = frf.sky_frates(V.hd, min_frate_width=1000, blkeys=to_filter)
+            # to_filter set to None -> all keys should be present.
+            if to_filter is None:
+                for k in V.data:
+                    assert k in cfrates
+                    assert k in wfrates
+            # min_frate = 1000 should set all wfrates to 1000
+            for k in cfrates:
+                assert wfrates[k] == 1000.
+
     def test_load_tophat_frfilter_and_write(self, tmpdir):
         tmp_path = tmpdir.strpath
         uvh5 = os.path.join(DATA_PATH, "test_input/zen.2458101.46106.xx.HH.OCR_53x_54x_only.uvh5")

scripts/tophat_frfilter_run.py

@@ -28,15 +28,14 @@
     filter_kwargs['skip_contiguous_flags'] = False
     filter_kwargs['skip_flagged_edges'] = False
     filter_kwargs['flag_model_rms_outliers'] = False
-elif ap.mode == 'dpss_leastsq':
+elif ap.mode == 'dpss_leastsq' or ap.mode == 'dft_leastsq':
     filter_kwargs = {}
     avg_red_bllens = True
     filter_kwargs['skip_contiguous_flags']=True
     filter_kwargs['skip_flagged_edges'] = True
     filter_kwargs['max_contiguous_edge_flags'] = 1
     filter_kwargs['flag_model_rms_outliers'] = True
 
-
 if ap.cornerturnfile is not None:
     baseline_list = io.baselines_from_filelist_position(filename=ap.cornerturnfile, filelist=ap.datafilelist)
 else:
@@ -60,6 +59,7 @@
                                    overwrite_flags=ap.overwrite_flags, skip_autos=ap.skip_autos,
                                    skip_if_flag_within_edge_distance=ap.skip_if_flag_within_edge_distance,
                                    zeropad=ap.zeropad, tol=ap.tol, skip_wgt=ap.skip_wgt, max_frate_coeffs=ap.max_frate_coeffs,
-                                   frate_width_multiplier=ap.frate_width_multiplier, frate_standoff=ap.frate_standoff,
-                                   min_frate_half_width=ap.min_frate_half_width,
-                                   clean_flags_in_resid_flags=True, **filter_kwargs)
+                                   frate_width_multiplier=ap.frate_width_multiplier, frate_standoff=ap.frate_standoff, fr_freq_skip=ap.fr_freq_skip,
+                                   min_frate_half_width=ap.min_frate_half_width, uvbeam=ap.uvbeam, percentile_low=ap.percentile_low, percentile_high=ap.percentile_high,
+                                   clean_flags_in_resid_flags=True, pre_filter_modes_between_lobe_minimum_and_zero=ap.pre_filter_modes_between_lobe_minimum_and_zero,
+                                   **filter_kwargs)

setup.py

@@ -41,7 +41,7 @@ def package_files(package_dir, subdirectory):
                 'scripts/smooth_cal_run.py', 'scripts/redcal_run.py',
                 'scripts/auto_reflection_run.py', 'scripts/noise_from_autos.py',
                 'scripts/query_ex_ants.py', 'scripts/red_average.py',
-                'scripts/time_average.py',
+                'scripts/time_average.py', 'scripts/tophat_frfilter_run.py',
                 'scripts/time_chunk_from_baseline_chunks_run.py', 'scripts/chunk_files.py', 'scripts/transfer_flags.py',
                 'scripts/flag_all.py', 'scripts/throw_away_flagged_antennas.py', 'scripts/select_spw_ranges.py'],
     'version': version.version,