diff --git a/.gitignore b/.gitignore index eca0bbf..e987388 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ dist build eggs +.eggs parts bin var @@ -28,6 +29,7 @@ pip-log.txt nosetests.xml *junit-* htmlcov +.pytest_cache # Translations *.mo @@ -42,4 +44,11 @@ output/*.html output/*/index.html # Sphinx -docs/_build \ No newline at end of file +docs/_build + +# Virtual environments +.virt +virt + +# Media +img \ No newline at end of file diff --git a/README.rst b/README.rst index 607eed8..2c8e042 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -============================= +==================================================== SEEK: Signal Extraction and Emission Kartographer -============================= +==================================================== .. image:: https://travis-ci.org/cosmo-ethz/seek.png?branch=master :target: https://travis-ci.org/cosmo-ethz/seek diff --git a/docs/conf.py b/docs/conf.py index 44a567c..480eea3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -129,7 +129,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/docs/mitigation.rst b/docs/mitigation.rst index 5b35778..1f014d3 100644 --- a/docs/mitigation.rst +++ b/docs/mitigation.rst @@ -3,7 +3,7 @@ RFI mitigation ================ SEEK's RFI mitigation follows the `Offringa et al. `_ `SumThreshold` algorithm. -It's implemented in pure Python and JIT-compiled for speed with the `HOPE `_ package. +It's implemented in pure Python. It is no longer JIT-compiled with the `HOPE `_ package as `HOPE` is now deprecated. It can easily be used without of the SEEK data processing pipeline:: diff --git a/docs/seek.rst b/docs/seek.rst index 349166c..bb881ca 100644 --- a/docs/seek.rst +++ b/docs/seek.rst @@ -1,23 +1,23 @@ -seek Package +seek package ============ -:mod:`seek` Package -------------------- - -.. automodule:: seek.__init__ - :members: - :undoc-members: - :show-inheritance: - Subpackages ----------- .. toctree:: + :maxdepth: 4 + + seek.calibration + seek.config + seek.mapmaking + seek.mitigation + seek.plugins + seek.utils - seek.calibration - seek.config - seek.mapmaking - seek.mitigation - seek.plugins - seek.utils +Module contents +--------------- +.. automodule:: seek + :members: + :undoc-members: + :show-inheritance: diff --git a/example_rfi.py b/example_rfi.py new file mode 100644 index 0000000..8af7ddb --- /dev/null +++ b/example_rfi.py @@ -0,0 +1,27 @@ +import cv2 as cv +import numpy.ma as ma + +from seek.mitigation import sum_threshold + +try: + spec = cv.imread("img/spec.png", cv.IMREAD_GRAYSCALE) + if spec is None: + raise FileNotFoundError() +except: + print("File not found. Try again.") + exit() + +spec_masked = ma.array(spec, mask=ma.nomask) + +rfi_mask = sum_threshold.get_rfi_mask( + tod=spec_masked, + mask=None, + chi_1=35000, + eta_i=[0.5], + #eta_i=[0.5, 0.55, 0.62, 0.75, 1], + normalize_standing_waves=True, + suppress_dilation=False, + plotting=True, + sm_kwargs=None, + di_kwargs=None +) \ No newline at end of file diff --git a/requirements-rfi.txt b/requirements-rfi.txt new file mode 100644 index 0000000..c5ee44e --- /dev/null +++ b/requirements-rfi.txt @@ -0,0 +1,12 @@ +# Requirements for the `seek.mitigation.get_rfi_mask` function +numpy==2.0.1 +scipy==1.14.0 + +# Requirements to run the example_rfi.py file +opencv-python==4.10.0.84 +matplotlib==3.9.1 + +# Requirements to successfully run the Makefile options +flake8 +pytest +sphinx diff --git a/requirements.readthedocs.txt b/requirements.readthedocs.txt index 1f2717d..4a0cd33 100644 --- a/requirements.readthedocs.txt +++ b/requirements.readthedocs.txt @@ -3,7 +3,6 @@ scipy Cython h5py mock -hope pyephem six astropy diff --git a/requirements.txt b/requirements.txt index 5dc648c..203dfae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ scipy Cython h5py mock -hope pyephem six healpy diff --git a/seek/calibration/fitting.py b/seek/calibration/fitting.py index eca2123..c3eae4b 100644 --- a/seek/calibration/fitting.py +++ b/seek/calibration/fitting.py @@ -22,9 +22,7 @@ import numpy as np from scipy.optimize import curve_fit -import hope -@hope.jit def gauss(x, a, x0, sigma, b, c): """ Gaussian model plus a linear background. diff --git a/seek/mitigation/sum_threshold.py b/seek/mitigation/sum_threshold.py index 66ceb40..6eb0b00 100644 --- a/seek/mitigation/sum_threshold.py +++ b/seek/mitigation/sum_threshold.py @@ -17,13 +17,14 @@ author: jakeret ''' + from __future__ import print_function, division, absolute_import, unicode_literals +import warnings + import numpy as np from scipy import ndimage -import hope - from seek.mitigation import sum_threshold_utils from seek.utils.tod_utils import get_empty_mask from seek.utils import filter @@ -41,7 +42,6 @@ STRUCT_SIZE = 3 -@hope.jit def _sumthreshold(data, mask, i, chi, ds0, ds1): """ The operation of summing and thresholding. @@ -111,7 +111,7 @@ def _run_sumthreshold(data, init_mask, eta, M, chi_i, sm_kwargs, plotting=True): st_mask = _sumthreshold(res.T, st_mask.T, m, chi, *res.T.shape).T if plotting: - sum_threshold_utils.plot_steps(data, st_mask, smoothed_data, res, "%s (%s)"%(eta, chi_i)) + sum_threshold_utils.plot_steps(data, st_mask, smoothed_data, res, f"{eta} ({chi_i})") return st_mask @@ -125,7 +125,7 @@ def binary_mask_dilation(mask, struct_size_0, struct_size_1): :return: dilated mask """ - struct = np.ones((struct_size_0, struct_size_1), np.bool) + struct = np.ones((struct_size_0, struct_size_1), np.bool_) return ndimage.binary_dilation(mask, structure=struct, iterations=2) @@ -159,19 +159,26 @@ def get_rfi_mask(tod, mask=None, chi_1=35000, eta_i=[0.5, 0.55, 0.62, 0.75, 1], :return mask: the mask covering the identified RFI """ + if mask is None and not suppress_dilation: + warnings.warn("mask=None and suppress_dilation=False: Dilation will not be performed on an empty mask.", UserWarning) + suppress_dilation = True + data = tod.data if mask is None: mask = get_empty_mask(data.shape) - if sm_kwargs is None: sm_kwargs = get_sm_kwargs() + if sm_kwargs is None: + sm_kwargs = get_sm_kwargs() - if plotting: sum_threshold_utils.plot_moments(data) + if plotting: + sum_threshold_utils.plot_moments(data, "Time and Frequency Statistics") if normalize_standing_waves: data = normalize(data, mask) - - if plotting: sum_threshold_utils.plot_moments(data) + if plotting: + title = "Time and Frequency Statistics After Normalizing Standing Waves" + sum_threshold_utils.plot_moments(data, title) p = 1.5 m = np.arange(1, MAX_PIXELS) @@ -184,13 +191,15 @@ def get_rfi_mask(tod, mask=None, chi_1=35000, eta_i=[0.5, 0.55, 0.62, 0.75, 1], dilated_mask = st_mask if not suppress_dilation: - if di_kwargs is None: di_kwargs = get_di_kwrags() + if di_kwargs is None: + di_kwargs = get_di_kwargs() - dilated_mask = binary_mask_dilation(dilated_mask - mask, **di_kwargs) + dilated_mask = binary_mask_dilation(np.logical_xor(dilated_mask, mask), **di_kwargs) - if plotting: sum_threshold_utils.plot_dilation(st_mask, mask, dilated_mask) + if plotting: + sum_threshold_utils.plot_dilation(st_mask, mask, dilated_mask) - return dilated_mask+mask + return dilated_mask + mask def get_sm_kwargs(kernel_m=KERNEL_M, kernel_n=KERNEL_N, sigma_m=SIGMA_M, sigma_n=SIGMA_N): """ @@ -205,7 +214,7 @@ def get_sm_kwargs(kernel_m=KERNEL_M, kernel_n=KERNEL_N, sigma_m=SIGMA_M, sigma_n """ return dict(M=kernel_m, N=kernel_n, sigma_m=sigma_m, sigma_n=sigma_n) -def get_di_kwrags(struct_size_0=STRUCT_SIZE, struct_size_1=STRUCT_SIZE): +def get_di_kwargs(struct_size_0=STRUCT_SIZE, struct_size_1=STRUCT_SIZE): """ Creates a dict with the dilation keywords. @@ -229,7 +238,7 @@ def get_sumthreshold_kwargs(params): params.sm_sigma_m, params.sm_sigma_n,) - di_kwargs = get_di_kwrags(params.struct_size_0, params.struct_size_1) + di_kwargs = get_di_kwargs(params.struct_size_0, params.struct_size_1) return sm_kwargs, di_kwargs diff --git a/seek/mitigation/sum_threshold_utils.py b/seek/mitigation/sum_threshold_utils.py index 4998a4b..7713688 100644 --- a/seek/mitigation/sum_threshold_utils.py +++ b/seek/mitigation/sum_threshold_utils.py @@ -20,6 +20,8 @@ from __future__ import print_function, division, absolute_import, unicode_literals import numpy as np +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1 import make_axes_locatable def get_stats(rfi, rfi_mask): """ @@ -38,82 +40,130 @@ def get_stats(rfi, rfi_mask): intersect = rfi_idx_set.intersection(mask_idx_set) return len(rfi_idx_set), len(mask_idx_set), len(intersect) - -def plot_data(data, ax, title, vmin=None, vmax=None, cb=True, norm=None, extent=None, cmap=None): + +def plot_data(data, ax, title, vmin=None, vmax=None, + cb=True, norm=None, extent=None, cmap=None): """ - Plot TOD. + Plot TOD with customization options. + + :param data: 2D array to be plotted + :param ax: matplotlib axis object where the plot will be drawn + :param title: title of the plot + :param vmin: minimum value for colormap scaling + :param vmax: maximum value for colormap scaling + :param cb: boolean flag to add a colorbar + :param norm: normalization for the colormap + :param extent: data limits for the axes + :param cmap: colormap to be used for the plot """ - import pylab - from mpl_toolkits.axes_grid1 import make_axes_locatable - + ax.set_title(title) - im = ax.imshow(data, - aspect="auto", - origin="lower", - norm=norm, - extent=extent, - cmap=cmap, - interpolation="nearest", vmin=vmin, vmax=vmax) - + + im = ax.imshow( + data, + aspect="auto", + origin="lower", + norm=norm, + extent=extent, + cmap=cmap, + interpolation="nearest", + vmin=vmin, + vmax=vmax, + ) + + # Add colorbar if flagged if cb: divider = make_axes_locatable(ax) cax = divider.append_axes("right", size="20%", pad=0.05) - cbar = pylab.colorbar(im, cax=cax) + plt.colorbar(im, cax=cax) + -def plot_moments(data): +def plot_moments(data, title): """ - Plot standard divation and mean of data. + Plot standard deviation and mean of data. """ - import pylab std_time = np.std(data, axis=0) mean_time = np.mean(data, axis=0) - std_freuqency = np.std(data, axis=1) - mean_freuqency = np.mean(data, axis=1) - pylab.subplot(121) - pylab.plot(mean_time) - pylab.xlabel("time") - pylab.ylabel("mean") - pylab.subplot(122) - pylab.plot(std_time) - pylab.xlabel("time") - pylab.ylabel("std") - pylab.show() - pylab.subplot(121) - pylab.plot(mean_freuqency) - pylab.xlabel("freuqency") - pylab.ylabel("mean") - pylab.subplot(122) - pylab.plot(std_freuqency) - pylab.xlabel("freuqency") - pylab.ylabel("std") - pylab.tight_layout() - pylab.show() + std_frequency = np.std(data, axis=1) + mean_frequency = np.mean(data, axis=1) + + fig, ax = plt.subplots(2, 2, figsize=(12, 8)) + + # Plot mean and std over time + ax[0, 0].plot(mean_time) + ax[0, 0].set_xlabel("Time") + ax[0, 0].set_ylabel("Mean") + ax[0, 0].set_title("Mean over Time") + + ax[0, 1].plot(std_time) + ax[0, 1].set_xlabel("Time") + ax[0, 1].set_ylabel("Standard Deviation") + ax[0, 1].set_title("Std Dev over Time") + + # Plot mean and std over frequency + ax[1, 0].plot(mean_frequency) + ax[1, 0].set_xlabel("Frequency") + ax[1, 0].set_ylabel("Mean") + ax[1, 0].set_title("Mean over Frequency") + + ax[1, 1].plot(std_frequency) + ax[1, 1].set_xlabel("Frequency") + ax[1, 1].set_ylabel("Standard Deviation") + ax[1, 1].set_title("Std Dev over Frequency") + + fig.suptitle(title, fontsize=16) + plt.tight_layout(rect=[0, 0, 1, 0.96]) # Adjust layout to make room for suptitle + + plt.tight_layout() + plt.show() + def plot_steps(data, st_mask, smoothed_data, res, eta): """ Plot individual steps of SumThreshold. + + :param data: Original data + :param st_mask: Sum threshold mask + :param smoothed_data: Smoothed data + :param res: Residuals + :param eta: Eta value """ - import pylab - f, ax = pylab.subplots(2,2, figsize=(15,8)) - f.suptitle("Eta: %s"%eta) - plot_data(data, ax[0,0], "data") - plot_data(st_mask, ax[1,0], "mask (%s)"%(st_mask.sum()), 0, 1) + + fig, ax = plt.subplots(2, 2, figsize=(15, 8)) + fig.suptitle(f"Eta: {eta}") + + plot_data(data, ax[0, 0], "Data") + plot_data(st_mask, ax[1, 0], f"Mask ({st_mask.sum()})", 0, 1) + smoothed = np.ma.MaskedArray(smoothed_data, st_mask) - plot_data(smoothed, ax[0,1], "_smooth") - plot_data(res, ax[1,1], "residuals") - f.show() + plot_data(smoothed, ax[0, 1], "Smoothed") + plot_data(res, ax[1, 1], "Residuals") + + plt.tight_layout(rect=[0, 0.03, 1, 0.95]) # Adjust layout to fit title + plt.show() def plot_dilation(st_mask, mask, dilated_mask): """ Plot mask and dilation. + + :param st_mask: Sum threshold mask + :param mask: Original mask + :param dilated_mask: Dilated mask """ - import pylab - fig, ax = pylab.subplots(2,2, figsize=(15,8)) + fig, ax = plt.subplots(2, 2, figsize=(15, 8)) fig.suptitle("Mask analysis") - plot_data(mask, ax[0,0], "Original mask") - plot_data(st_mask.astype(np.bool)-mask, ax[0,1], "Sum threshold mask", 0, 1) - plot_data(dilated_mask, ax[1,0], "dilated mask") - plot_data(dilated_mask+mask, ax[1,1], "New mask") - fig.show() + + plot_data(mask, ax[0, 0], "Original mask") + plot_data( + np.logical_xor(st_mask.astype(np.bool_), mask), + ax[0, 1], + "Sum threshold mask", + 0, + 1, + ) + plot_data(dilated_mask, ax[1, 0], "dilated mask") + plot_data(dilated_mask + mask, ax[1, 1], "New mask") + + plt.show() diff --git a/seek/plugins/background_removal.py b/seek/plugins/background_removal.py index 63b32f3..54cf91a 100644 --- a/seek/plugins/background_removal.py +++ b/seek/plugins/background_removal.py @@ -24,7 +24,7 @@ from ivy.plugin.base_plugin import BasePlugin from seek.mitigation import sum_threshold import healpy as hp -from scipy.ndimage.filters import gaussian_filter as gaussian +from scipy.ndimage import gaussian_filter as gaussian def mask_galaxy(nside, mask_original, mask_gal, ra, dec): @@ -77,7 +77,7 @@ def __call__(self): sigma_m = 2, sigma_n = 25) - di_kwargs = sum_threshold.get_di_kwrags(self.ctx.params.struct_size_0, + di_kwargs = sum_threshold.get_di_kwargs(self.ctx.params.struct_size_0, self.ctx.params.struct_size_1) mask_gal = self.ctx.simulation_mask diff --git a/seek/utils/filter.py b/seek/utils/filter.py index ea27881..bfb23d6 100644 --- a/seek/utils/filter.py +++ b/seek/utils/filter.py @@ -20,7 +20,6 @@ from __future__ import print_function, division, absolute_import, unicode_literals import numpy as np -import hope def gaussian_filter(V, mask, M=40, N=20, sigma_m=0.5, sigma_n=0.5): """ @@ -56,11 +55,10 @@ def wd(n, m, sigma_n, sigma_m): Vh[mask] = V[mask] return Vh -@hope.jit def _gaussian_filter(Vp, vs0, vs1, Wfp, mask, Vh, Vh2, kernel_0, kernel_1, M, N): - n2 = N/2 - m2 = M/2 + n2 = N//2 + m2 = M//2 for i in range((N//2), vs0+(N//2)): for j in range((M//2), vs1+(M//2)): if mask[i-n2, j-m2]: diff --git a/seek/utils/tod_utils.py b/seek/utils/tod_utils.py index ed64b45..d3a098d 100644 --- a/seek/utils/tod_utils.py +++ b/seek/utils/tod_utils.py @@ -21,11 +21,8 @@ import numpy as np -import hope - def get_empty_mask(shape): - mask = np.empty(shape, dtype=np.bool) - mask[:] = False + mask = np.full(shape, False, dtype=np.bool_) return mask def smooth(tod, factor, axis = 1): @@ -48,7 +45,6 @@ def smooth(tod, factor, axis = 1): smooth_tod = tod[:, :nnew*factor].reshape(m, nnew, factor) return smooth_tod.mean(axis = 2) -@hope.jit def spectral_kurtosis(p, p2, M, offset): """ Computes the spectral kurtosis for the given P and P^2 values diff --git a/setup.py b/setup.py index 1a22727..321a5c3 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def run_tests(self): history = open('HISTORY.rst').read().replace('.. :changelog:', '') -requires = ["ivy", "hope", "numpy", "scipy", "h5py", "pyephem", "astropy"] +requires = ["ivy-wfengine", "numpy", "scipy", "h5py", "pyephem", "astropy"] tests_require=['pytest>=2.3', "mock"] PACKAGE_PATH = os.path.abspath(os.path.join(__file__, os.pardir))