diff --git a/docs/source/user_manual/results_filtering.rst b/docs/source/user_manual/results_filtering.rst index f2ea3b48..44589de6 100644 --- a/docs/source/user_manual/results_filtering.rst +++ b/docs/source/user_manual/results_filtering.rst @@ -21,29 +21,6 @@ Relevant light curve filtering parameters include: * ``sigmaG_lims`` - The percentiles for sigmaG filtering (default of [25, 75]). -Stamp Filtering ---------------- - -The stamp filtering stage is only applied if the ``do_stamp_filter`` parameter is set to True. This stage creates a single stamp representing the sum, mean, median, or variance weighted mean of pixel values for the stamps at each time step. The stamp type is defined by the ``stamp_type`` parameter and can take on values ``median``, ``mean``, ``sum``, or ``weighted`` (for variance weighted). All of the stamp types drop masked pixels from their computations. The mean and median sums are computed over only the valid time steps from the light curve filtering phase (dropping stamps with outlier fluxes). The sum coadd uses all the time steps regardless of the first phase of filtering. - -The stamps are filtered based on how closely the pixel values in the stamp image represent a Gaussian defined with the parameters: - -* ``center_thresh`` - The percentage of flux in the central pixel. For example setting this to 0.9 will require that the central pixel of the stamp has 90 percent of all the flux in the stamp. -* ``peak_offset`` - How far the brightest pixel is from the center of the stamp (in pixels). For example a peak offset of [2.0, 3.0] requires that the brightest pixel in the stamp is at most 2 pixels from the center in the x-dimension and 3-pixels from the center in the y-dimension. -* ``mom_lims`` - Compute the Gaussian moments of the image and compares them to the thresholds. - -Relevant stamp filtering parameters include: - - * ``center_thresh`` - The percentage of flux in the central pixel. - * ``chunk_size`` - The number of candidate trajectories to filter in a batch. Used to control memory usage. - * ``do_stamp_filter`` - A Boolean indicating whether to generate and filter stamps. - * ``peak_offset`` - A length 2 list indicating how far the peak is from the center of the stamp in each of the x and y dimensions. - * ``mom_lims`` - A length 5 list providing thresholds on the images moments. - * ``stamp_radius`` - The radius of the stamps to use. - -Note that stamps are only generated and output into files if ``do_stamp_filter`` is set to true. - - Clustering ---------- diff --git a/docs/source/user_manual/search_params.rst b/docs/source/user_manual/search_params.rst index 64cf5e37..25690d39 100644 --- a/docs/source/user_manual/search_params.rst +++ b/docs/source/user_manual/search_params.rst @@ -33,11 +33,6 @@ Configuration Parameters +------------------------+-----------------------------+----------------------------------------+ | **Parameter** | **Default Value** | **Interpretation** | +------------------------+-----------------------------+----------------------------------------+ -| ``center_thresh`` | 0.00 | The minimum fraction of total flux | -| | | within a stamp that must be contained | -| | | in the central pixel | -| | | (if ``do_stamp_filter=True``). | -+------------------------+-----------------------------+----------------------------------------+ | ``chunk_size`` | 500000 | The batch size to use when processing | | | | the results of the on-GPU search. | +------------------------+-----------------------------+----------------------------------------+ @@ -76,9 +71,6 @@ Configuration Parameters +------------------------+-----------------------------+----------------------------------------+ | ``do_mask`` | True | Apply the mask to the raw pixels. | +------------------------+-----------------------------+----------------------------------------+ -| ``do_stamp_filter`` | True | Apply post-search filtering on the | -| | | image stamps. | -+------------------------+-----------------------------+----------------------------------------+ | ``encode_num_bytes`` | -1 | The number of bytes to use to encode | | | | ``psi`` and ``phi`` images on GPU. By | | | | default a ``float`` encoding is used. | @@ -113,20 +105,10 @@ Configuration Parameters | | | computed likelihood above this | | | | threshold are rejected. | +------------------------+-----------------------------+----------------------------------------+ -| ``mom_lims`` | [35.5, 35.5, 2.0, 0.3, 0.3] | Thresholds for the moments of a | -| | | Gaussian fit to the flux, specified as | -| | | ``[xx, yy, xy, x, y]``. | -| | | If ``do_stamp_filter=True``. | -+------------------------+-----------------------------+----------------------------------------+ | ``num_obs`` | 10 | The minimum number of non-masked | | | | observations for the object to be | | | | accepted. | +------------------------+-----------------------------+----------------------------------------+ -| ``peak_offset`` | [2.0, 2.0] | How far, in pixels, the brightest pixel| -| | | in the stamp can be from the central | -| | | pixel in each direction ``[x,y]``. | -| | | If ``do_stamp_filter=True``). | -+------------------------+-----------------------------+----------------------------------------+ | ``psf_val`` | 1.4 | The value for the standard deviation of| | | | the point spread function (PSF). | +------------------------+-----------------------------+----------------------------------------+ @@ -148,9 +130,8 @@ Configuration Parameters | | | creating a stamp for stamp filtering | | | | (in pixels). | +------------------------+-----------------------------+----------------------------------------+ -| ``stamp_type`` | sum | The type of coadd to use during stamp | -| | | filtering (if ``do_stamp_filter=True``)| -| | | if: | +| ``stamp_type`` | sum | The type of coadd to use as the main | +| | | stamp: | | | | * ``sum`` - (default) Per pixel sum | | | | * ``median`` - Per pixel median | | | | * ``mean`` - Per pixel mean | diff --git a/src/kbmod/configuration.py b/src/kbmod/configuration.py index 7580af10..af232243 100644 --- a/src/kbmod/configuration.py +++ b/src/kbmod/configuration.py @@ -17,7 +17,6 @@ def __init__(self): self._required_params = set() self._params = { - "center_thresh": 0.00, "chunk_size": 500000, "clip_negative": False, "cluster_eps": 20.0, @@ -27,7 +26,6 @@ def __init__(self): "debug": False, "do_clustering": True, "do_mask": True, - "do_stamp_filter": True, "encode_num_bytes": -1, "generator_config": { "name": "EclipticCenteredSearch", @@ -41,9 +39,7 @@ def __init__(self): "im_filepath": None, "lh_level": 10.0, "max_lh": 1000.0, - "mom_lims": [35.5, 35.5, 2.0, 0.3, 0.3], "num_obs": 10, - "peak_offset": [2.0, 2.0], "psf_val": 1.4, "result_filename": None, "results_per_pixel": 8, diff --git a/src/kbmod/filters/stamp_filters.py b/src/kbmod/filters/stamp_filters.py index 0fd7d08d..2cd572bd 100644 --- a/src/kbmod/filters/stamp_filters.py +++ b/src/kbmod/filters/stamp_filters.py @@ -63,29 +63,10 @@ def extract_search_parameters_from_config(config): else: raise ValueError(f"Unrecognized stamp type: {stamp_type}") - # Filtering parameters (with validity checking) - params.do_filtering = config["do_stamp_filter"] - params.center_thresh = config["center_thresh"] - - peak_offset = config["peak_offset"] - if len(peak_offset) != 2: - raise ValueError(f"Expected length 2 list for peak_offset. Found {peak_offset}") - params.peak_offset_x = peak_offset[0] - params.peak_offset_y = peak_offset[1] - - mom_lims = config["mom_lims"] - if len(mom_lims) != 5: - raise ValueError(f"Expected length 5 list for mom_lims. Found {mom_lims}") - params.m20_limit = mom_lims[0] - params.m02_limit = mom_lims[1] - params.m11_limit = mom_lims[2] - params.m10_limit = mom_lims[3] - params.m01_limit = mom_lims[4] - return params -def get_coadds_and_filter_results(result_data, im_stack, stamp_params, chunk_size=1_000_000, colname="stamp"): +def make_coadds(result_data, im_stack, stamp_params, chunk_size=1_000_000, colname="stamp"): """Create the co-added postage stamps and filter them based on their statistical properties. Results with stamps that are similar to a Gaussian are kept. @@ -157,17 +138,12 @@ def get_coadds_and_filter_results(result_data, im_stack, stamp_params, chunk_siz # collecting RawImage but leaving a dangling ref to the attribute. # That's a fix for another time so I'm leaving it as a copy here for ind, stamp in enumerate(stamps_slice): - if stamp.width > 1: - stamps_to_keep.append(np.array(stamp.image)) - keep_row[start_idx + ind] = True + stamps_to_keep.append(np.array(stamp.image)) + keep_row[start_idx + ind] = True # Move to the next chunk. start_idx += chunk_size - # Do the actual filtering of results - if stamp_params.do_filtering: - result_data.filter_rows(keep_row, label="stamp_filter") - # Append the coadded stamps to the results. We do this after the filtering # so we are not adding a jagged array. result_data.table[colname] = np.array(stamps_to_keep) @@ -195,7 +171,6 @@ def append_coadds(result_data, im_stack, coadd_types, radius, chunk_size=100_000 params = StampParameters() params.radius = radius - params.do_filtering = False # Loop through all the coadd types in the list, generating a corresponding stamp. for coadd_type in coadd_types: @@ -213,7 +188,7 @@ def append_coadds(result_data, im_stack, coadd_types, radius, chunk_size=100_000 raise ValueError(f"Unrecognized stamp type: {coadd_type}") # Do the generation (without filtering). - get_coadds_and_filter_results( + make_coadds( result_data, im_stack, params, diff --git a/src/kbmod/run_search.py b/src/kbmod/run_search.py index 85c7417a..e4ca80af 100644 --- a/src/kbmod/run_search.py +++ b/src/kbmod/run_search.py @@ -9,7 +9,12 @@ from .configuration import SearchConfiguration from .filters.clustering_filters import apply_clustering from .filters.sigma_g_filter import apply_clipped_sigma_g, SigmaGClipping -from .filters.stamp_filters import append_all_stamps, append_coadds, get_coadds_and_filter_results +from .filters.stamp_filters import ( + append_all_stamps, + append_coadds, + extract_search_parameters_from_config, + make_coadds, +) from .results import Results from .trajectory_generator import create_trajectory_generator @@ -238,11 +243,6 @@ def run_search( trj_generator = create_trajectory_generator(config, work_unit=None) keep = self.do_gpu_search(config, stack, trj_generator) - if config["do_stamp_filter"]: - stack.copy_to_gpu() - get_coadds_and_filter_results(keep, stack, config) - stack.clear_from_gpu() - if config["do_clustering"]: cluster_timer = kb.DebugTimer("clustering", logger) mjds = [stack.get_obstime(t) for t in range(stack.img_count())] @@ -255,12 +255,14 @@ def run_search( apply_clustering(keep, cluster_params) cluster_timer.stop() - # Generate additional coadded stamps without filtering. + # Generate coadded stamps without filtering -- both the "stamp" column + # as well as any additional coadds. + stamp_params = extract_search_parameters_from_config(config) + make_coadds(keep, stack, stamp_params, colname="stamp") if len(config["coadds"]) > 0: stack.copy_to_gpu() append_coadds(keep, stack, config["coadds"], config["stamp_radius"]) stack.clear_from_gpu() - logger.info(f"Found {len(keep)} potential trajectories.") # Extract all the stamps for all time steps and append them onto the result rows. if config["save_all_stamps"]: diff --git a/src/kbmod/search/bindings.cpp b/src/kbmod/search/bindings.cpp index 4e797dfa..cff7103e 100644 --- a/src/kbmod/search/bindings.cpp +++ b/src/kbmod/search/bindings.cpp @@ -42,7 +42,6 @@ PYBIND11_MODULE(search, m) { search::stack_search_bindings(m); search::stamp_creator_bindings(m); search::trajectory_bindings(m); - search::image_moments_bindings(m); search::stamp_parameters_bindings(m); search::psi_phi_array_binding(m); search::debug_timer_binding(m); diff --git a/src/kbmod/search/common.h b/src/kbmod/search/common.h index 3c609b27..5f005efc 100644 --- a/src/kbmod/search/common.h +++ b/src/kbmod/search/common.h @@ -142,50 +142,9 @@ struct SearchParameters { struct StampParameters { int radius = 10; StampType stamp_type = STAMP_SUM; - bool do_filtering = false; - - // Thresholds on the location of the image peak. - float center_thresh; - float peak_offset_x; - float peak_offset_y; - - // Limits on the moments. - float m01_limit; - float m10_limit; - float m11_limit; - float m02_limit; - float m20_limit; - - const std::string to_string() const { - // If filtering is turned off, output the minimal information on a single line. - // Otherwise dump the full statistics on multiple lines. - if (!do_filtering) { - return ("Type: " + std::to_string(stamp_type) + " Radius: " + std::to_string(radius) + - " Filtering: false"); - } else { - return ("Type: " + std::to_string(stamp_type) + "\nRadius: " + std::to_string(radius) + - "\nFiltering: true" + "\nCenter Thresh: " + std::to_string(center_thresh) + - "\nPeak Offset: x=" + std::to_string(peak_offset_x) + " y=" + - std::to_string(peak_offset_y) + "\nMoment Limits: m01=" + std::to_string(m01_limit) + - " m10=" + std::to_string(m10_limit) + " m11=" + std::to_string(m11_limit) + - " m02=" + std::to_string(m02_limit) + " m20=" + std::to_string(m02_limit)); - } - } -}; - -// Basic image moments use for analysis. -struct ImageMoments { - float m00; - float m01; - float m10; - float m11; - float m02; - float m20; const std::string to_string() const { - return "m00: " + std::to_string(m00) + " m01: " + std::to_string(m01) + - " m10: " + std::to_string(m10) + " m11: " + std::to_string(m11) + - " m02: " + std::to_string(m02) + " m20: " + std::to_string(m20); + return ("Type: " + std::to_string(stamp_type) + " Radius: " + std::to_string(radius)); } }; @@ -225,33 +184,12 @@ static void trajectory_bindings(py::module &m) { })); } -static void image_moments_bindings(py::module &m) { - py::class_(m, "ImageMoments", pydocs::DOC_ImageMoments) - .def(py::init<>()) - .def("__str__", &ImageMoments::to_string) - .def_readwrite("m00", &ImageMoments::m00) - .def_readwrite("m01", &ImageMoments::m01) - .def_readwrite("m10", &ImageMoments::m10) - .def_readwrite("m11", &ImageMoments::m11) - .def_readwrite("m02", &ImageMoments::m02) - .def_readwrite("m20", &ImageMoments::m20); -} - static void stamp_parameters_bindings(py::module &m) { py::class_(m, "StampParameters", pydocs::DOC_StampParameters) .def(py::init<>()) .def("__str__", &StampParameters::to_string) .def_readwrite("radius", &StampParameters::radius) - .def_readwrite("stamp_type", &StampParameters::stamp_type) - .def_readwrite("do_filtering", &StampParameters::do_filtering) - .def_readwrite("center_thresh", &StampParameters::center_thresh) - .def_readwrite("peak_offset_x", &StampParameters::peak_offset_x) - .def_readwrite("peak_offset_y", &StampParameters::peak_offset_y) - .def_readwrite("m01_limit", &StampParameters::m01_limit) - .def_readwrite("m10_limit", &StampParameters::m10_limit) - .def_readwrite("m11_limit", &StampParameters::m11_limit) - .def_readwrite("m02_limit", &StampParameters::m02_limit) - .def_readwrite("m20_limit", &StampParameters::m20_limit); + .def_readwrite("stamp_type", &StampParameters::stamp_type); } #endif /* Py_PYTHON_H */ diff --git a/src/kbmod/search/pydocs/raw_image_docs.h b/src/kbmod/search/pydocs/raw_image_docs.h index 6ee784b6..b8c8bbbe 100644 --- a/src/kbmod/search/pydocs/raw_image_docs.h +++ b/src/kbmod/search/pydocs/raw_image_docs.h @@ -205,50 +205,6 @@ static const auto DOC_RawImage_compute_bounds = R"doc( A ``(min, max)`` tuple. )doc"; -static const auto DOC_RawImage_find_peak = R"doc( - Returns the pixel coordinates of the maximum value. - - Parameters - ---------- - furthest_from_center : `bool` - When `True`, and multiple identical maxima exist, returns the one that is - at a greatest L2 distance from the center of the image. Otherwise it - returns the last maxima that was found in a row-wise ordered search. - - Returns - ------- - location : `Index`, optional - Index of the maximum. - )doc"; - -static const auto DOC_RawImage_find_central_moments = R"doc( - Returns the central moments of the image. - - Returns - ------- - moments : `ImageMoments` - Image moments. - )doc"; - -static const auto DOC_RawImage_center_is_local_max = R"doc( - A filter on whether the center of the stamp is a local - maxima and the percentage of the stamp's total flux in this - pixel. - - Parameters - ---------- - local_max : ``bool`` - Require the central pixel to be a local maximum. - flux_thresh : ``float`` - The fraction of the stamp's total flux that needs to be in - the center pixel [0.0, 1.0]. - - Returns - ------- - keep_row : `bool` - Whether or not the stamp passes the check. - )doc"; - static const auto DOC_RawImage_create_stamp = R"doc( Create an image stamp around a given region. diff --git a/src/kbmod/search/pydocs/stamp_creator_docs.h b/src/kbmod/search/pydocs/stamp_creator_docs.h index de0a0bea..a28d8c12 100644 --- a/src/kbmod/search/pydocs/stamp_creator_docs.h +++ b/src/kbmod/search/pydocs/stamp_creator_docs.h @@ -141,25 +141,6 @@ static const auto DOC_StampCreator_get_coadded_stamps = R"doc( )doc"; -static const auto DOC_StampCreator_filter_stamp = R"doc( - Filters stamps based on the given parameters. - - Applies the following filters: peak position, percent flux at central pixel, - and image moments. - - Parameters - ---------- - img : `RawImage` - The image to test. - params : `StampParameters` - The parameters for stamp generation and filtering. - - Returns - ------- - `bool` - Whether or not to filter the stamp. - )doc"; - static const auto DOC_StampCreator_create_variance_stamps = R"doc( Create a vector of stamps from the variance layer centered on the predicted position of an Trajectory at different times. diff --git a/src/kbmod/search/raw_image.cpp b/src/kbmod/search/raw_image.cpp index 15357f46..836741f1 100644 --- a/src/kbmod/search/raw_image.cpp +++ b/src/kbmod/search/raw_image.cpp @@ -184,106 +184,6 @@ void RawImage::apply_mask(int flags, const RawImage& mask) { void RawImage::set_all(float value) { image.setConstant(value); } -// The maximum value of the image and return the coordinates. -Index RawImage::find_peak(bool furthest_from_center) const { - int c_x = width / 2; - int c_y = height / 2; - - // Initialize the variables for tracking the peak's location. - Index result = {0, 0}; - float max_val = std::numeric_limits::lowest(); - float dist2 = c_x * c_x + c_y * c_y; - - // Search each pixel for the peak. - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - float pix_val = image(y, x); - if (pixel_value_valid(pix_val) && (pix_val > max_val)) { - max_val = pix_val; - result.i = y; - result.j = x; - dist2 = (c_x - x) * (c_x - x) + (c_y - y) * (c_y - y); - } else if (pixel_value_valid(pix_val) && (pix_val == max_val)) { - int new_dist2 = (c_x - x) * (c_x - x) + (c_y - y) * (c_y - y); - if ((furthest_from_center && (new_dist2 > dist2)) || - (!furthest_from_center && (new_dist2 < dist2))) { - max_val = pix_val; - result.i = y; - result.j = x; - dist2 = new_dist2; - } - } - } // for x - } // for y - return result; -} - -// Find the basic image moments in order to test if stamps have a gaussian shape. -// It computes the moments on the "normalized" image where the minimum -// value has been shifted to zero and the sum of all elements is 1.0. -// Elements with invalid or masked data are treated as zero. -ImageMoments RawImage::find_central_moments() const { - const uint64_t num_pixels = width * height; - const int c_x = width / 2; - const int c_y = height / 2; - - // Set all the moments to zero initially. - ImageMoments res = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - auto pixels = image.reshaped(); - - // Find the minimum (valid) value to subtract off. - float min_val = FLT_MAX; - for (uint64_t p = 0; p < num_pixels; ++p) { - min_val = (pixel_value_valid(pixels[p]) && (pixels[p] < min_val)) ? pixels[p] : min_val; - } - - // Find the sum of the zero-shifted (valid) pixels. - double sum = 0.0; - for (uint64_t p = 0; p < num_pixels; ++p) { - sum += pixel_value_valid(pixels[p]) ? (pixels[p] - min_val) : 0.0; - } - if (sum == 0.0) return res; - - // Compute the rest of the moments. - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - int ind = y * width + x; - float pix_val = pixel_value_valid(pixels[ind]) ? (pixels[ind] - min_val) / sum : 0.0; - - res.m00 += pix_val; - res.m10 += (x - c_x) * pix_val; - res.m20 += (x - c_x) * (x - c_x) * pix_val; - res.m01 += (y - c_y) * pix_val; - res.m02 += (y - c_y) * (y - c_y) * pix_val; - res.m11 += (x - c_x) * (y - c_y) * pix_val; - } - } - - return res; -} - -bool RawImage::center_is_local_max(double flux_thresh, bool local_max) const { - const uint64_t num_pixels = width * height; - int c_x = width / 2; - int c_y = height / 2; - int c_ind = c_y * width + c_x; - - auto pixels = image.reshaped(); - double center_val = pixels[c_ind]; - - // Find the sum of the zero-shifted (valid) pixels. - double sum = 0.0; - for (uint64_t p = 0; p < num_pixels; ++p) { - float pix_val = pixels[p]; - if (p != c_ind && local_max && pix_val >= center_val) { - return false; - } - sum += pixel_value_valid(pixels[p]) ? pix_val : 0.0; - } - if (sum == 0.0) return false; - return center_val / sum >= flux_thresh; -} - // it makes no sense to return RawImage here because there is no // obstime by definition of operation, but I guess it's out of // scope for this PR because it requires updating layered_image @@ -447,10 +347,6 @@ static void raw_image_bindings(py::module& m) { pydocs::DOC_RawImage_replace_masked_values) .def("compute_bounds", &rie::compute_bounds, py::arg("strict_checks") = true, pydocs::DOC_RawImage_compute_bounds) - .def("find_peak", &rie::find_peak, pydocs::DOC_RawImage_find_peak) - .def("find_central_moments", &rie::find_central_moments, - pydocs::DOC_RawImage_find_central_moments) - .def("center_is_local_max", &rie::center_is_local_max, pydocs::DOC_RawImage_center_is_local_max) .def("create_stamp", &rie::create_stamp, pydocs::DOC_RawImage_create_stamp) .def("apply_mask", &rie::apply_mask, pydocs::DOC_RawImage_apply_mask) .def("convolve_gpu", &rie::convolve, pydocs::DOC_RawImage_convolve_gpu) diff --git a/src/kbmod/search/raw_image.h b/src/kbmod/search/raw_image.h index 143b4dd6..b73564f2 100644 --- a/src/kbmod/search/raw_image.h +++ b/src/kbmod/search/raw_image.h @@ -94,19 +94,6 @@ class RawImage { // to apply (use 0xFFFFFF to apply all flags). void apply_mask(int flags, const RawImage& mask); - // The maximum value of the image and return the coordinates. The parameter - // furthest_from_center indicates whether to break ties using the peak further - // or closer to the center of the image. - Index find_peak(bool furthest_from_center) const; - - // Find the basic image moments in order to test if stamps have a gaussian shape. - // It computes the moments on the "normalized" image where the minimum - // value has been shifted to zero and the sum of all elements is 1.0. - // Elements with NO_DATA, NaN, etc. are treated as zero. - ImageMoments find_central_moments() const; - - bool center_is_local_max(double flux_thresh, bool local_max) const; - virtual ~RawImage(){}; private: diff --git a/src/kbmod/search/stamp_creator.cpp b/src/kbmod/search/stamp_creator.cpp index 83202a3a..ffa74eb4 100644 --- a/src/kbmod/search/stamp_creator.cpp +++ b/src/kbmod/search/stamp_creator.cpp @@ -112,61 +112,14 @@ std::vector get_coadded_stamps_cpu(ImageStack& stack, std::vector get_coadded_stamps_gpu(ImageStack& stack, std::vector results(num_trajectories); std::vector current_pixels(stamp_ppi, 0.0); for (uint64_t t = 0; t < num_trajectories; ++t) { @@ -269,12 +222,7 @@ std::vector get_coadded_stamps_gpu(ImageStack& stack, std::vector(current_pixels.data(), stamp_width, stamp_width); RawImage current_image = RawImage(tmp); - - if (params.do_filtering && filter_stamp(current_image, params)) { - results[t] = std::move(RawImage(1, 1, NO_DATA)); - } else { - results[t] = std::move(current_image); - } + results[t] = std::move(current_image); } return results; } @@ -353,7 +301,6 @@ static void stamp_creator_bindings(py::module& m) { m.def("create_stamps", &search::create_stamps, pydocs::DOC_StampCreator_create_stamps); m.def("create_variance_stamps", &search::create_variance_stamps, pydocs::DOC_StampCreator_create_variance_stamps); - m.def("filter_stamp", &search::filter_stamp, pydocs::DOC_StampCreator_filter_stamp); } #endif /* Py_PYTHON_H */ diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 1c955f28..37723234 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -30,11 +30,15 @@ def test_demo_defaults(self): # Load the WorkUnit. input_data = WorkUnit.from_fits(filename) + input_data.config.set("coadds", ["mean"]) rs = SearchRunner() keep = rs.run_search_from_work_unit(input_data) self.assertGreaterEqual(len(keep), 1) + + # Check that we have both a stamp column and a coadd_mean column. self.assertEqual(keep["stamp"][0].shape, (21, 21)) + self.assertEqual(keep["coadd_mean"][0].shape, (21, 21)) @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") def test_demo_stamp_size(self): @@ -48,8 +52,8 @@ def test_demo_stamp_size(self): # Override the stamp settings of the configuration input_data.config.set("stamp_radius", 15) - input_data.config.set("mom_lims", [80.0, 80.0, 50.0, 20.0, 20.0]) input_data.config.set("save_all_stamps", True) + input_data.config.set("coadds", ["mean"]) rs = SearchRunner() keep = rs.run_search_from_work_unit(input_data) @@ -58,6 +62,9 @@ def test_demo_stamp_size(self): self.assertIsNotNone(keep["stamp"][0]) self.assertEqual(keep["stamp"][0].shape, (31, 31)) + self.assertIsNotNone(keep["coadd_mean"][0]) + self.assertEqual(keep["coadd_mean"][0].shape, (31, 31)) + self.assertIsNotNone(keep["all_stamps"][0]) for s in keep["all_stamps"][0]: self.assertEqual(s.shape, (31, 31)) diff --git a/tests/test_raw_image.py b/tests/test_raw_image.py index 6af507a5..fa0fce5c 100644 --- a/tests/test_raw_image.py +++ b/tests/test_raw_image.py @@ -177,75 +177,6 @@ def test_replace_masked_values(self): else: self.assertEqual(img2.get_pixel(row, col), 0.0) - def test_find_peak(self): - "Test RawImage find_peak" - img = RawImage(self.masked_array) - idx = img.find_peak(False) - self.assertEqual(idx.i, 5) - self.assertEqual(idx.j, 5) - - # We found the peak furthest to the center. - idx = img.find_peak(True) - self.assertEqual(idx.i, 3) - self.assertEqual(idx.j, 1) - - # We are okay when the data includes NaNs. - img.set_pixel(2, 3, math.nan) - img.set_pixel(3, 2, np.nan) - idx = img.find_peak(False) - self.assertEqual(idx.i, 5) - self.assertEqual(idx.j, 5) - - def test_find_central_moments(self): - """Test RawImage central moments.""" - img = RawImage(5, 5, value=0.1) - - # Try something mostly symmetric and centered. - img.set_pixel(2, 2, 10.0) - img.set_pixel(2, 1, 5.0) - img.set_pixel(1, 2, 5.0) - img.set_pixel(2, 3, 5.0) - img.set_pixel(3, 2, 5.0) - - img_mom = img.find_central_moments() - self.assertAlmostEqual(img_mom.m00, 1.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m01, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m10, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m11, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m02, 0.3322, delta=1e-4) - self.assertAlmostEqual(img_mom.m20, 0.3322, delta=1e-4) - - # Try something flat symmetric and centered. - img.set_all(2.0) - img_mom = img.find_central_moments() - - self.assertAlmostEqual(img_mom.m00, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m01, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m10, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m11, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m02, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m20, 0.0, delta=1e-4) - - # Try something with a few non-symmetric peaks. - img.set_all(0.4) - img.set_pixel(2, 2, 5.0) - img.set_pixel(0, 1, 5.0) - img.set_pixel(3, 3, 10.0) - img.set_pixel(0, 3, 0.2) - img_mom = img.find_central_moments() - - self.assertAlmostEqual(img_mom.m00, 1.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m01, 0.20339, delta=1e-4) - self.assertAlmostEqual(img_mom.m10, 0.03390, delta=1e-4) - self.assertAlmostEqual(img_mom.m11, 0.81356, delta=1e-4) - self.assertAlmostEqual(img_mom.m02, 1.01695, delta=1e-4) - self.assertAlmostEqual(img_mom.m20, 1.57627, delta=1e-4) - - # Check that nothing fails with NaNs. - img.set_pixel(2, 3, math.nan) - img.set_pixel(3, 2, np.nan) - img_mom = img.find_central_moments() - def convolve_psf_identity(self, device): psf_data = np.zeros((3, 3), dtype=np.single) psf_data[1, 1] = 1.0 diff --git a/tests/test_search.py b/tests/test_search.py index d55c9616..843f12ef 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -83,16 +83,7 @@ def setUp(self): # Set the filtering parameters. self.params = StampParameters() self.params.radius = 5 - self.params.do_filtering = True self.params.stamp_type = StampType.STAMP_MEAN - self.params.center_thresh = 0.03 - self.params.peak_offset_x = 1.5 - self.params.peak_offset_y = 1.5 - self.params.m01_limit = 1.0 - self.params.m10_limit = 1.0 - self.params.m11_limit = 2.0 - self.params.m02_limit = 35.5 - self.params.m20_limit = 35.5 self.trj_gen = KBMODV1Search( self.velocity_steps, diff --git a/tests/test_stamp_creator.py b/tests/test_stamp_creator.py index 65186f5d..5e71f620 100644 --- a/tests/test_stamp_creator.py +++ b/tests/test_stamp_creator.py @@ -25,7 +25,6 @@ get_variance_weighted_stamp, create_stamps, create_variance_stamps, - filter_stamp, ) @@ -96,16 +95,7 @@ def setUp(self): # Set the filtering parameters. self.params = StampParameters() self.params.radius = 5 - self.params.do_filtering = True self.params.stamp_type = StampType.STAMP_MEAN - self.params.center_thresh = 0.03 - self.params.peak_offset_x = 1.5 - self.params.peak_offset_y = 1.5 - self.params.m01_limit = 1.0 - self.params.m10_limit = 1.0 - self.params.m11_limit = 2.0 - self.params.m02_limit = 35.5 - self.params.m20_limit = 35.5 # Create a second (smaller) fake data set to use in some of the tests. self.img_count2 = 10 @@ -372,53 +362,6 @@ def test_mean_stamps_no_data(self): # Check that we get the correct answer. self.assertAlmostEqual(pix_sum / pix_count, meanStamp.get_pixel(2, 2), delta=1e-5) - def test_filter_stamp(self): - stamp_width = 2 * self.params.radius + 1 - - # Test a stamp with nothing in it. - stamp = RawImage(stamp_width, stamp_width) - stamp.set_all(1.0) - self.assertTrue(filter_stamp(stamp, self.params)) - - # Test a stamp with a bright spot in the center. - stamp.set_pixel(5, 5, 100.0) - self.assertFalse(filter_stamp(stamp, self.params)) - - # A little noise around the pixel does not hurt as long as the shape is - # roughly Gaussian. - stamp.set_pixel(4, 5, 15.0) - stamp.set_pixel(5, 4, 10.0) - stamp.set_pixel(6, 5, 10.0) - stamp.set_pixel(5, 6, 20.0) - self.assertFalse(filter_stamp(stamp, self.params)) - - # A bright peak far from the center is bad. - stamp.set_pixel(1, 1, 500.0) - self.assertTrue(filter_stamp(stamp, self.params)) - stamp.set_pixel(1, 1, 1.0) - - # A non-Gaussian bright spot is also bad. Blur to the -x direction. - stamp.set_pixel(4, 5, 50.0) - stamp.set_pixel(3, 5, 50.0) - stamp.set_pixel(2, 5, 60.0) - stamp.set_pixel(4, 4, 45.0) - stamp.set_pixel(3, 4, 45.0) - stamp.set_pixel(2, 4, 55.0) - stamp.set_pixel(4, 6, 55.0) - stamp.set_pixel(3, 6, 55.0) - stamp.set_pixel(2, 6, 65.0) - self.assertTrue(filter_stamp(stamp, self.params)) - - # A very dim peak at the center is invalid. - stamp.set_all(1.0) - stamp.set_pixel(5, 5, 1.0001) - self.assertTrue(filter_stamp(stamp, self.params)) - - # A slightly offset peak of sufficient brightness is okay. - stamp.set_pixel(5, 5, 15.0) - stamp.set_pixel(4, 5, 20.0) - self.assertFalse(filter_stamp(stamp, self.params)) - def test_coadd_cpu_simple(self): # Create an image set with three images. imlist = [] @@ -456,7 +399,6 @@ def test_coadd_cpu_simple(self): # Basic Stamp parameters. params = StampParameters() params.radius = 1 - params.do_filtering = False # Test summed. params.stamp_type = StampType.STAMP_SUM @@ -529,7 +471,6 @@ def test_coadd_gpu_simple(self): # Basic Stamp parameters. params = StampParameters() params.radius = 1 - params.do_filtering = False # Test summed. params.stamp_type = StampType.STAMP_SUM @@ -555,7 +496,6 @@ def test_coadd_gpu_simple(self): def test_coadd_cpu(self): params = StampParameters() params.radius = 3 - params.do_filtering = False # Compute the stacked science (summed and mean) from a single Trajectory. params.stamp_type = StampType.STAMP_SUM @@ -606,7 +546,6 @@ def test_coadd_cpu(self): def test_coadd_gpu(self): params = StampParameters() params.radius = 3 - params.do_filtering = False # Compute the stacked science (summed and mean) from a single Trajectory. params.stamp_type = StampType.STAMP_SUM @@ -671,7 +610,6 @@ def test_coadd_gpu_large(self): for radius in [10, 20, 30]: params = StampParameters() params.radius = radius - params.do_filtering = False # Test that we get valid stamp values for all pixels (not all zeros). params.stamp_type = StampType.STAMP_MEAN @@ -699,7 +637,6 @@ def test_coadd_cpu_fallback(self): # Use a radius that is too large for the GPU. params = StampParameters() params.radius = 500 - params.do_filtering = False # Test that we get valid stamp values for all pixels. params.stamp_type = StampType.STAMP_MEAN @@ -713,7 +650,6 @@ def test_coadd_cpu_fallback(self): def test_coadd_cpu_use_inds(self): params = StampParameters() params.radius = 1 - params.do_filtering = False params.stamp_type = StampType.STAMP_MEAN # Mark a few of the observations as "do not use" @@ -762,7 +698,6 @@ def test_coadd_cpu_use_inds(self): def test_coadd_gpu_use_inds(self): params = StampParameters() params.radius = 1 - params.do_filtering = False params.stamp_type = StampType.STAMP_MEAN # Mark a few of the observations as "do not use" @@ -807,83 +742,6 @@ def test_coadd_gpu_use_inds(self): self.assertAlmostEqual(sum_0 / count_0, meanStamps[0].get_pixel(stamp_y, stamp_x), delta=1e-3) self.assertAlmostEqual(sum_1 / count_1, meanStamps[1].get_pixel(stamp_y, stamp_x), delta=1e-3) - def test_coadd_filter_cpu(self): - # Create a second Trajectory that isn't any good. - trj2 = Trajectory(x=1, y=1, vx=0.0, vy=0.0) - - # Create a third Trajectory that is close to good, but offset. - trj3 = Trajectory( - x=self.trj.x + 2, - y=self.trj.y + 2, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Create a fourth Trajectory that is close enough - trj4 = Trajectory( - x=self.trj.x + 1, - y=self.trj.y + 1, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Compute the stacked science from a single Trajectory. - all_valid_vect = [(self.all_valid) for i in range(4)] - meanStamps = get_coadded_stamps( - self.stack, [self.trj, trj2, trj3, trj4], all_valid_vect, self.params, True - ) - - # The first and last are unfiltered - self.assertEqual(meanStamps[0].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[0].height, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].height, 2 * self.params.radius + 1) - - # The second and third are filtered. - self.assertEqual(meanStamps[1].width, 1) - self.assertEqual(meanStamps[1].height, 1) - self.assertEqual(meanStamps[2].width, 1) - self.assertEqual(meanStamps[2].height, 1) - - @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") - def test_coadd_filter_gpu(self): - # Create a second Trajectory that isn't any good. - trj2 = Trajectory(x=1, y=1, vx=0.0, vy=0.0) - - # Create a third Trajectory that is close to good, but offset. - trj3 = Trajectory( - x=self.trj.x + 2, - y=self.trj.y + 2, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Create a fourth Trajectory that is close enough - trj4 = Trajectory( - x=self.trj.x + 1, - y=self.trj.y + 1, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Compute the stacked science from a single Trajectory. - all_valid_vect = [(self.all_valid) for i in range(4)] - meanStamps = get_coadded_stamps( - self.stack, [self.trj, trj2, trj3, trj4], all_valid_vect, self.params, True - ) - - # The first and last are unfiltered - self.assertEqual(meanStamps[0].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[0].height, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].height, 2 * self.params.radius + 1) - - # The second and third are filtered. - self.assertEqual(meanStamps[1].width, 1) - self.assertEqual(meanStamps[1].height, 1) - self.assertEqual(meanStamps[2].width, 1) - self.assertEqual(meanStamps[2].height, 1) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_stamp_filters.py b/tests/test_stamp_filters.py index 9f3e71e1..4ad82676 100644 --- a/tests/test_stamp_filters.py +++ b/tests/test_stamp_filters.py @@ -28,10 +28,6 @@ def setUp(self): def test_extract_search_parameters_from_config(self): config_dict = { - "center_thresh": 0.05, - "do_stamp_filter": True, - "mom_lims": [50.0, 51.0, 1.0, 2.0, 3.0], - "peak_offset": [1.5, 3.5], "stamp_type": "median", "stamp_radius": 7, } @@ -40,15 +36,6 @@ def test_extract_search_parameters_from_config(self): params = extract_search_parameters_from_config(config) self.assertEqual(params.radius, 7) self.assertEqual(params.stamp_type, StampType.STAMP_MEDIAN) - self.assertEqual(params.do_filtering, True) - self.assertAlmostEqual(params.center_thresh, 0.05) - self.assertAlmostEqual(params.peak_offset_x, 1.5) - self.assertAlmostEqual(params.peak_offset_y, 3.5) - self.assertAlmostEqual(params.m20_limit, 50.0) - self.assertAlmostEqual(params.m02_limit, 51.0) - self.assertAlmostEqual(params.m11_limit, 1.0) - self.assertAlmostEqual(params.m10_limit, 2.0) - self.assertAlmostEqual(params.m01_limit, 3.0) # Test bad configurations config.set("stamp_radius", -1) @@ -59,16 +46,8 @@ def test_extract_search_parameters_from_config(self): self.assertRaises(ValueError, extract_search_parameters_from_config, config) config.set("stamp_type", "median") - config.set("peak_offset", [50.0]) - self.assertRaises(ValueError, extract_search_parameters_from_config, config) - config.set("peak_offset", [1.5, 3.5]) - - config.set("mom_lims", [50.0, 51.0, 1.0, 3.0]) - self.assertRaises(ValueError, extract_search_parameters_from_config, config) - config.set("mom_lims", [50.0, 51.0, 1.0, 2.0, 3.0]) - @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") - def test_get_coadds_and_filter_results(self): + def test_make_coadds(self): # Create trajectories to test: 0) known good, 1) completely wrong # 2) close to good, but offset], and 3) just close enough. trj_list = [ @@ -82,25 +61,18 @@ def test_get_coadds_and_filter_results(self): # Create the stamp parameters we need. config_dict = { - "center_thresh": 0.03, - "do_stamp_filter": True, - "mom_lims": [35.5, 35.5, 1.0, 1.0, 1.0], - "peak_offset": [1.5, 1.5], "stamp_type": "cpp_mean", "stamp_radius": 5, } config = SearchConfiguration.from_dict(config_dict) - # Do the filtering. - get_coadds_and_filter_results(keep, self.ds.stack, config, chunk_size=2) + # Make the stamps. + make_coadds(keep, self.ds.stack, config, chunk_size=2) - # The check that the correct indices and number of stamps are saved. + # We do not filter, so everything should be saved. self.assertTrue("stamp" in keep.colnames) - self.assertEqual(len(keep), 2) - self.assertEqual(keep["x"][0], self.trj.x) - self.assertEqual(keep["x"][1], self.trj.x + 1) + self.assertEqual(len(keep), 4) self.assertEqual(keep["stamp"][0].shape, (11, 11)) - self.assertEqual(keep["stamp"][1].shape, (11, 11)) @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") def test_get_coadds_and_filter_with_invalid(self): @@ -119,27 +91,19 @@ def test_get_coadds_and_filter_with_invalid(self): # Create the stamp parameters we need. config_dict = { - "center_thresh": 0.03, - "do_stamp_filter": True, - "mom_lims": [35.5, 35.5, 1.0, 1.0, 1.0], - "peak_offset": [1.5, 1.5], "stamp_type": "cpp_mean", "stamp_radius": 5, } config = SearchConfiguration.from_dict(config_dict) - # Do the filtering. - get_coadds_and_filter_results(keep, self.ds.stack, config, chunk_size=2) - - # The check that the correct indices and number of stamps are saved. + # Make the stamps and check that there were saved. + make_coadds(keep, self.ds.stack, config, chunk_size=2) self.assertTrue("stamp" in keep.colnames) - self.assertEqual(len(keep), 1) - self.assertEqual(keep["vx"][0], trj2.vx) - self.assertEqual(keep["vy"][0], trj2.vy) + self.assertEqual(len(keep), 2) # Test with empty results. keep2 = Results.from_trajectories([]) - get_coadds_and_filter_results(keep2, self.ds.stack, config, chunk_size=1000) + make_coadds(keep2, self.ds.stack, config, chunk_size=1000) self.assertTrue("stamp" in keep2.colnames) @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") diff --git a/tests/test_stamp_parity.py b/tests/test_stamp_parity.py index dc584ed8..ef4fdb11 100644 --- a/tests/test_stamp_parity.py +++ b/tests/test_stamp_parity.py @@ -52,7 +52,6 @@ def test_coadd_gpu_parity(self): width = 2 * radius + 1 params = StampParameters() params.radius = radius - params.do_filtering = False results = [self.trj, self.trj] all_valid = [1] * self.img_count