From 8cb9cb72602887a7fa2cf9eecbe91bd015e80015 Mon Sep 17 00:00:00 2001 From: Jeremy Kubica <104161096+jeremykubica@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:54:22 -0500 Subject: [PATCH 1/4] Remove stamp filtering from KBMOD --- src/kbmod/configuration.py | 4 - src/kbmod/filters/stamp_filters.py | 33 +---- src/kbmod/run_search.py | 7 +- src/kbmod/search/bindings.cpp | 1 - src/kbmod/search/common.h | 66 +-------- src/kbmod/search/pydocs/raw_image_docs.h | 44 ------ src/kbmod/search/pydocs/stack_search_docs.h | 2 +- src/kbmod/search/pydocs/stamp_creator_docs.h | 19 --- .../search/pydocs/trajectory_list_docs.h | 1 - src/kbmod/search/raw_image.cpp | 114 +------------- src/kbmod/search/raw_image.h | 13 -- src/kbmod/search/stamp_creator.cpp | 62 +------- src/kbmod/search/stamp_creator.h | 6 +- tests/test_end_to_end.py | 9 +- tests/test_raw_image.py | 69 --------- tests/test_search.py | 9 -- tests/test_stamp_creator.py | 139 ------------------ tests/test_stamp_filters.py | 54 ++----- tests/test_stamp_parity.py | 1 - 19 files changed, 30 insertions(+), 623 deletions(-) diff --git a/src/kbmod/configuration.py b/src/kbmod/configuration.py index f406d8dce..a6cac8a28 100644 --- a/src/kbmod/configuration.py +++ b/src/kbmod/configuration.py @@ -19,7 +19,6 @@ def __init__(self): self._required_params = set() self._params = { - "center_thresh": 0.00, "chunk_size": 500000, "clip_negative": False, "cluster_eps": 20.0, @@ -29,7 +28,6 @@ def __init__(self): "debug": False, "do_clustering": True, "do_mask": True, - "do_stamp_filter": True, "encode_num_bytes": -1, "generator_config": { "name": "EclipticCenteredSearch", @@ -44,9 +42,7 @@ def __init__(self): "legacy_filename": None, "lh_level": 10.0, "max_lh": 1000.0, - "mom_lims": [35.5, 35.5, 2.0, 0.3, 0.3], "num_obs": 10, - "peak_offset": [2.0, 2.0], "psf_val": 1.4, "result_filename": None, "results_per_pixel": 8, diff --git a/src/kbmod/filters/stamp_filters.py b/src/kbmod/filters/stamp_filters.py index c4f537e8f..486d3dc7f 100644 --- a/src/kbmod/filters/stamp_filters.py +++ b/src/kbmod/filters/stamp_filters.py @@ -62,29 +62,10 @@ def extract_search_parameters_from_config(config): else: raise ValueError(f"Unrecognized stamp type: {stamp_type}") - # Filtering parameters (with validity checking) - params.do_filtering = config["do_stamp_filter"] - params.center_thresh = config["center_thresh"] - - peak_offset = config["peak_offset"] - if len(peak_offset) != 2: - raise ValueError(f"Expected length 2 list for peak_offset. Found {peak_offset}") - params.peak_offset_x = peak_offset[0] - params.peak_offset_y = peak_offset[1] - - mom_lims = config["mom_lims"] - if len(mom_lims) != 5: - raise ValueError(f"Expected length 5 list for mom_lims. Found {mom_lims}") - params.m20_limit = mom_lims[0] - params.m02_limit = mom_lims[1] - params.m11_limit = mom_lims[2] - params.m10_limit = mom_lims[3] - params.m01_limit = mom_lims[4] - return params -def get_coadds_and_filter_results(result_data, im_stack, stamp_params, chunk_size=1_000_000, colname="stamp"): +def make_coadds(result_data, im_stack, stamp_params, chunk_size=1_000_000, colname="stamp"): """Create the co-added postage stamps and filter them based on their statistical properties. Results with stamps that are similar to a Gaussian are kept. @@ -156,17 +137,12 @@ def get_coadds_and_filter_results(result_data, im_stack, stamp_params, chunk_siz # collecting RawImage but leaving a dangling ref to the attribute. # That's a fix for another time so I'm leaving it as a copy here for ind, stamp in enumerate(stamps_slice): - if stamp.width > 1: - stamps_to_keep.append(np.array(stamp.image)) - keep_row[start_idx + ind] = True + stamps_to_keep.append(np.array(stamp.image)) + keep_row[start_idx + ind] = True # Move to the next chunk. start_idx += chunk_size - # Do the actual filtering of results - if stamp_params.do_filtering: - result_data.filter_rows(keep_row, label="stamp_filter") - # Append the coadded stamps to the results. We do this after the filtering # so we are not adding a jagged array. result_data.table[colname] = np.array(stamps_to_keep) @@ -194,7 +170,6 @@ def append_coadds(result_data, im_stack, coadd_types, radius, chunk_size=100_000 params = StampParameters() params.radius = radius - params.do_filtering = False # Loop through all the coadd types in the list, generating a corresponding stamp. for coadd_type in coadd_types: @@ -212,7 +187,7 @@ def append_coadds(result_data, im_stack, coadd_types, radius, chunk_size=100_000 raise ValueError(f"Unrecognized stamp type: {coadd_type}") # Do the generation (without filtering). - get_coadds_and_filter_results( + make_coadds( result_data, im_stack, params, diff --git a/src/kbmod/run_search.py b/src/kbmod/run_search.py index faf21cede..8320f7a51 100644 --- a/src/kbmod/run_search.py +++ b/src/kbmod/run_search.py @@ -8,7 +8,7 @@ from .configuration import SearchConfiguration from .filters.clustering_filters import apply_clustering from .filters.sigma_g_filter import apply_clipped_sigma_g, SigmaGClipping -from .filters.stamp_filters import append_all_stamps, append_coadds, get_coadds_and_filter_results +from .filters.stamp_filters import append_all_stamps, append_coadds from .results import Results from .trajectory_generator import create_trajectory_generator @@ -223,11 +223,6 @@ def run_search(self, config, stack, trj_generator=None, wcs=None): trj_generator = create_trajectory_generator(config, work_unit=None) keep = self.do_gpu_search(config, stack, trj_generator) - if config["do_stamp_filter"]: - stack.copy_to_gpu() - get_coadds_and_filter_results(keep, stack, config) - stack.clear_from_gpu() - if config["do_clustering"]: cluster_timer = kb.DebugTimer("clustering", logger) mjds = [stack.get_obstime(t) for t in range(stack.img_count())] diff --git a/src/kbmod/search/bindings.cpp b/src/kbmod/search/bindings.cpp index b068252e6..361e89eab 100644 --- a/src/kbmod/search/bindings.cpp +++ b/src/kbmod/search/bindings.cpp @@ -42,7 +42,6 @@ PYBIND11_MODULE(search, m) { search::stack_search_bindings(m); search::stamp_creator_bindings(m); search::trajectory_bindings(m); - search::image_moments_bindings(m); search::stamp_parameters_bindings(m); search::psi_phi_array_binding(m); search::debug_timer_binding(m); diff --git a/src/kbmod/search/common.h b/src/kbmod/search/common.h index 3c609b279..5f005efcc 100644 --- a/src/kbmod/search/common.h +++ b/src/kbmod/search/common.h @@ -142,50 +142,9 @@ struct SearchParameters { struct StampParameters { int radius = 10; StampType stamp_type = STAMP_SUM; - bool do_filtering = false; - - // Thresholds on the location of the image peak. - float center_thresh; - float peak_offset_x; - float peak_offset_y; - - // Limits on the moments. - float m01_limit; - float m10_limit; - float m11_limit; - float m02_limit; - float m20_limit; - - const std::string to_string() const { - // If filtering is turned off, output the minimal information on a single line. - // Otherwise dump the full statistics on multiple lines. - if (!do_filtering) { - return ("Type: " + std::to_string(stamp_type) + " Radius: " + std::to_string(radius) + - " Filtering: false"); - } else { - return ("Type: " + std::to_string(stamp_type) + "\nRadius: " + std::to_string(radius) + - "\nFiltering: true" + "\nCenter Thresh: " + std::to_string(center_thresh) + - "\nPeak Offset: x=" + std::to_string(peak_offset_x) + " y=" + - std::to_string(peak_offset_y) + "\nMoment Limits: m01=" + std::to_string(m01_limit) + - " m10=" + std::to_string(m10_limit) + " m11=" + std::to_string(m11_limit) + - " m02=" + std::to_string(m02_limit) + " m20=" + std::to_string(m02_limit)); - } - } -}; - -// Basic image moments use for analysis. -struct ImageMoments { - float m00; - float m01; - float m10; - float m11; - float m02; - float m20; const std::string to_string() const { - return "m00: " + std::to_string(m00) + " m01: " + std::to_string(m01) + - " m10: " + std::to_string(m10) + " m11: " + std::to_string(m11) + - " m02: " + std::to_string(m02) + " m20: " + std::to_string(m20); + return ("Type: " + std::to_string(stamp_type) + " Radius: " + std::to_string(radius)); } }; @@ -225,33 +184,12 @@ static void trajectory_bindings(py::module &m) { })); } -static void image_moments_bindings(py::module &m) { - py::class_(m, "ImageMoments", pydocs::DOC_ImageMoments) - .def(py::init<>()) - .def("__str__", &ImageMoments::to_string) - .def_readwrite("m00", &ImageMoments::m00) - .def_readwrite("m01", &ImageMoments::m01) - .def_readwrite("m10", &ImageMoments::m10) - .def_readwrite("m11", &ImageMoments::m11) - .def_readwrite("m02", &ImageMoments::m02) - .def_readwrite("m20", &ImageMoments::m20); -} - static void stamp_parameters_bindings(py::module &m) { py::class_(m, "StampParameters", pydocs::DOC_StampParameters) .def(py::init<>()) .def("__str__", &StampParameters::to_string) .def_readwrite("radius", &StampParameters::radius) - .def_readwrite("stamp_type", &StampParameters::stamp_type) - .def_readwrite("do_filtering", &StampParameters::do_filtering) - .def_readwrite("center_thresh", &StampParameters::center_thresh) - .def_readwrite("peak_offset_x", &StampParameters::peak_offset_x) - .def_readwrite("peak_offset_y", &StampParameters::peak_offset_y) - .def_readwrite("m01_limit", &StampParameters::m01_limit) - .def_readwrite("m10_limit", &StampParameters::m10_limit) - .def_readwrite("m11_limit", &StampParameters::m11_limit) - .def_readwrite("m02_limit", &StampParameters::m02_limit) - .def_readwrite("m20_limit", &StampParameters::m20_limit); + .def_readwrite("stamp_type", &StampParameters::stamp_type); } #endif /* Py_PYTHON_H */ diff --git a/src/kbmod/search/pydocs/raw_image_docs.h b/src/kbmod/search/pydocs/raw_image_docs.h index 6a3e8c407..93c6002e3 100644 --- a/src/kbmod/search/pydocs/raw_image_docs.h +++ b/src/kbmod/search/pydocs/raw_image_docs.h @@ -198,50 +198,6 @@ static const auto DOC_RawImage_compute_bounds = R"doc( bounds : `tuple` A ``(min, max)`` tuple. )doc"; - -static const auto DOC_RawImage_find_peak = R"doc( - Returns the pixel coordinates of the maximum value. - - Parameters - ---------- - furthest_from_center : `bool` - When `True`, and multiple identical maxima exist, returns the one that is - at a greatest L2 distance from the center of the image. Otherwise it - returns the last maxima that was found in a row-wise ordered search. - - Returns - ------- - location : `Index`, optional - Index of the maximum. - )doc"; - -static const auto DOC_RawImage_find_central_moments = R"doc( - Returns the central moments of the image. - - Returns - ------- - moments : `ImageMoments` - Image moments. - )doc"; - -static const auto DOC_RawImage_center_is_local_max = R"doc( - A filter on whether the center of the stamp is a local - maxima and the percentage of the stamp's total flux in this - pixel. - - Parameters - ---------- - local_max : ``bool`` - Require the central pixel to be a local maximum. - flux_thresh : ``float`` - The fraction of the stamp's total flux that needs to be in - the center pixel [0.0, 1.0]. - - Returns - ------- - keep_row : `bool` - Whether or not the stamp passes the check. - )doc"; static const auto DOC_RawImage_create_stamp = R"doc( Create an image stamp around a given region. diff --git a/src/kbmod/search/pydocs/stack_search_docs.h b/src/kbmod/search/pydocs/stack_search_docs.h index bf9c48783..e64c030d5 100644 --- a/src/kbmod/search/pydocs/stack_search_docs.h +++ b/src/kbmod/search/pydocs/stack_search_docs.h @@ -169,7 +169,7 @@ static const auto DOC_StackSearch_get_number_total_results = R"doc( result : `int` The number of cached results. )doc"; - + static const auto DOC_StackSearch_get_results = R"doc( Get a batch of cached results. diff --git a/src/kbmod/search/pydocs/stamp_creator_docs.h b/src/kbmod/search/pydocs/stamp_creator_docs.h index 1b648397b..e374472df 100644 --- a/src/kbmod/search/pydocs/stamp_creator_docs.h +++ b/src/kbmod/search/pydocs/stamp_creator_docs.h @@ -146,25 +146,6 @@ static const auto DOC_StampCreator_get_coadded_stamps = R"doc( )doc"; -static const auto DOC_StampCreator_filter_stamp = R"doc( - Filters stamps based on the given parameters. - - Applies the following filters: peak position, percent flux at central pixel, - and image moments. - - Parameters - ---------- - img : `RawImage` - The image to test. - params : `StampParameters` - The parameters for stamp generation and filtering. - - Returns - ------- - `bool` - Whether or not to filter the stamp. - )doc"; - static const auto DOC_StampCreator_create_variance_stamps = R"doc( Create a vector of stamps from the variance layer centered on the predicted position of an Trajectory at different times. diff --git a/src/kbmod/search/pydocs/trajectory_list_docs.h b/src/kbmod/search/pydocs/trajectory_list_docs.h index 23ae5f43f..0afeaabd6 100644 --- a/src/kbmod/search/pydocs/trajectory_list_docs.h +++ b/src/kbmod/search/pydocs/trajectory_list_docs.h @@ -143,7 +143,6 @@ static const auto DOC_TrajectoryList_sort_by_likelihood = R"doc( Raises a ``RuntimeError`` the data is on GPU. )doc"; - } // namespace pydocs #endif /* TRAJECTORY_LIST_DOCS_ */ diff --git a/src/kbmod/search/raw_image.cpp b/src/kbmod/search/raw_image.cpp index 226e54dfe..e9389f4e4 100644 --- a/src/kbmod/search/raw_image.cpp +++ b/src/kbmod/search/raw_image.cpp @@ -12,8 +12,7 @@ RawImage::RawImage(Image& img) { width = image.cols(); } -RawImage::RawImage(unsigned w, unsigned h, float value) - : width(w), height(h) { +RawImage::RawImage(unsigned w, unsigned h, float value) : width(w), height(h) { if (value != 0.0f) image = Image::Constant(height, width, value); else @@ -29,9 +28,7 @@ RawImage::RawImage(const RawImage& old) noexcept { // Move constructor RawImage::RawImage(RawImage&& source) noexcept - : width(source.width), - height(source.height), - image(std::move(source.image)) {} + : width(source.width), height(source.height), image(std::move(source.image)) {} // Copy assignment RawImage& RawImage::operator=(const RawImage& source) noexcept { @@ -245,106 +242,6 @@ void RawImage::apply_mask(int flags, const RawImage& mask) { void RawImage::set_all(float value) { image.setConstant(value); } -// The maximum value of the image and return the coordinates. -Index RawImage::find_peak(bool furthest_from_center) const { - int c_x = width / 2; - int c_y = height / 2; - - // Initialize the variables for tracking the peak's location. - Index result = {0, 0}; - float max_val = std::numeric_limits::lowest(); - float dist2 = c_x * c_x + c_y * c_y; - - // Search each pixel for the peak. - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - float pix_val = image(y, x); - if (pixel_value_valid(pix_val) && (pix_val > max_val)) { - max_val = pix_val; - result.i = y; - result.j = x; - dist2 = (c_x - x) * (c_x - x) + (c_y - y) * (c_y - y); - } else if (pixel_value_valid(pix_val) && (pix_val == max_val)) { - int new_dist2 = (c_x - x) * (c_x - x) + (c_y - y) * (c_y - y); - if ((furthest_from_center && (new_dist2 > dist2)) || - (!furthest_from_center && (new_dist2 < dist2))) { - max_val = pix_val; - result.i = y; - result.j = x; - dist2 = new_dist2; - } - } - } // for x - } // for y - return result; -} - -// Find the basic image moments in order to test if stamps have a gaussian shape. -// It computes the moments on the "normalized" image where the minimum -// value has been shifted to zero and the sum of all elements is 1.0. -// Elements with invalid or masked data are treated as zero. -ImageMoments RawImage::find_central_moments() const { - const uint64_t num_pixels = width * height; - const int c_x = width / 2; - const int c_y = height / 2; - - // Set all the moments to zero initially. - ImageMoments res = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - auto pixels = image.reshaped(); - - // Find the minimum (valid) value to subtract off. - float min_val = FLT_MAX; - for (uint64_t p = 0; p < num_pixels; ++p) { - min_val = (pixel_value_valid(pixels[p]) && (pixels[p] < min_val)) ? pixels[p] : min_val; - } - - // Find the sum of the zero-shifted (valid) pixels. - double sum = 0.0; - for (uint64_t p = 0; p < num_pixels; ++p) { - sum += pixel_value_valid(pixels[p]) ? (pixels[p] - min_val) : 0.0; - } - if (sum == 0.0) return res; - - // Compute the rest of the moments. - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - int ind = y * width + x; - float pix_val = pixel_value_valid(pixels[ind]) ? (pixels[ind] - min_val) / sum : 0.0; - - res.m00 += pix_val; - res.m10 += (x - c_x) * pix_val; - res.m20 += (x - c_x) * (x - c_x) * pix_val; - res.m01 += (y - c_y) * pix_val; - res.m02 += (y - c_y) * (y - c_y) * pix_val; - res.m11 += (x - c_x) * (y - c_y) * pix_val; - } - } - - return res; -} - -bool RawImage::center_is_local_max(double flux_thresh, bool local_max) const { - const uint64_t num_pixels = width * height; - int c_x = width / 2; - int c_y = height / 2; - int c_ind = c_y * width + c_x; - - auto pixels = image.reshaped(); - double center_val = pixels[c_ind]; - - // Find the sum of the zero-shifted (valid) pixels. - double sum = 0.0; - for (uint64_t p = 0; p < num_pixels; ++p) { - float pix_val = pixels[p]; - if (p != c_ind && local_max && pix_val >= center_val) { - return false; - } - sum += pixel_value_valid(pixels[p]) ? pix_val : 0.0; - } - if (sum == 0.0) return false; - return center_val / sum >= flux_thresh; -} - // it makes no sense to return RawImage here because there is no // obstime by definition of operation, but I guess it's out of // scope for this PR because it requires updating layered_image @@ -460,8 +357,7 @@ static void raw_image_bindings(py::module& m) { .def(py::init<>()) .def(py::init()) .def(py::init(), py::arg("img").noconvert(true)) - .def(py::init(), py::arg("w"), py::arg("h"), - py::arg("value") = 0.0f) + .def(py::init(), py::arg("w"), py::arg("h"), py::arg("value") = 0.0f) // attributes and properties .def_property_readonly("height", &rie::get_height) .def_property_readonly("width", &rie::get_width) @@ -508,10 +404,6 @@ static void raw_image_bindings(py::module& m) { .def("replace_masked_values", &rie::replace_masked_values, py::arg("value") = 0.0f, pydocs::DOC_RawImage_replace_masked_values) .def("compute_bounds", &rie::compute_bounds, pydocs::DOC_RawImage_compute_bounds) - .def("find_peak", &rie::find_peak, pydocs::DOC_RawImage_find_peak) - .def("find_central_moments", &rie::find_central_moments, - pydocs::DOC_RawImage_find_central_moments) - .def("center_is_local_max", &rie::center_is_local_max, pydocs::DOC_RawImage_center_is_local_max) .def("create_stamp", &rie::create_stamp, pydocs::DOC_RawImage_create_stamp) .def("interpolate", &rie::interpolate, pydocs::DOC_RawImage_interpolate) .def("interpolated_add", &rie::interpolated_add, pydocs::DOC_RawImage_interpolated_add) diff --git a/src/kbmod/search/raw_image.h b/src/kbmod/search/raw_image.h index 9fe52a774..e4c7eb1fc 100644 --- a/src/kbmod/search/raw_image.h +++ b/src/kbmod/search/raw_image.h @@ -104,19 +104,6 @@ class RawImage { // to apply (use 0xFFFFFF to apply all flags). void apply_mask(int flags, const RawImage& mask); - // The maximum value of the image and return the coordinates. The parameter - // furthest_from_center indicates whether to break ties using the peak further - // or closer to the center of the image. - Index find_peak(bool furthest_from_center) const; - - // Find the basic image moments in order to test if stamps have a gaussian shape. - // It computes the moments on the "normalized" image where the minimum - // value has been shifted to zero and the sum of all elements is 1.0. - // Elements with NO_DATA, NaN, etc. are treated as zero. - ImageMoments find_central_moments() const; - - bool center_is_local_max(double flux_thresh, bool local_max) const; - virtual ~RawImage(){}; private: diff --git a/src/kbmod/search/stamp_creator.cpp b/src/kbmod/search/stamp_creator.cpp index c5d1341af..02d014bbe 100644 --- a/src/kbmod/search/stamp_creator.cpp +++ b/src/kbmod/search/stamp_creator.cpp @@ -105,60 +105,12 @@ std::vector StampCreator::get_coadded_stamps_cpu(ImageStack& stack, default: throw std::runtime_error("Invalid stamp coadd type."); } - - // Do the filtering if needed. - if (params.do_filtering && filter_stamp(coadd, params)) { - results[i] = std::move(RawImage(1, 1, NO_DATA)); - } else { - results[i] = std::move(coadd); - } + results[i] = std::move(coadd); } return results; } -bool StampCreator::filter_stamp(const RawImage& img, const StampParameters& params) { - if (params.radius <= 0) throw std::runtime_error("Invalid stamp radius=" + std::to_string(params.radius)); - - // Allocate space for the coadd information and initialize to zero. - const unsigned int stamp_width = 2 * params.radius + 1; - const uint64_t stamp_ppi = stamp_width * stamp_width; - // this ends up being something like eigen::vector1f something, not vector - // but it behaves in all the same ways so just let it figure it out itself - const auto& pixels = img.get_image().reshaped(); - - // Filter on the peak's position. - Index idx = img.find_peak(true); - if ((abs(idx.i - params.radius) >= params.peak_offset_x) || - (abs(idx.j - params.radius) >= params.peak_offset_y)) { - return true; - } - - // Filter on the percentage of flux in the central pixel. - if (params.center_thresh > 0.0) { - const auto& pixels = img.get_image().reshaped(); - float center_val = pixels[idx.j * stamp_width + idx.i]; - float pixel_sum = 0.0; - for (uint64_t p = 0; p < stamp_ppi; ++p) { - pixel_sum += pixels[p]; - } - - if (center_val / pixel_sum < params.center_thresh) { - return true; - } - } - - // Filter on the image moments. - ImageMoments moments = img.find_central_moments(); - if ((fabs(moments.m01) >= params.m01_limit) || (fabs(moments.m10) >= params.m10_limit) || - (fabs(moments.m11) >= params.m11_limit) || (moments.m02 >= params.m02_limit) || - (moments.m20 >= params.m20_limit)) { - return true; - } - - return false; -} - std::vector StampCreator::get_coadded_stamps_gpu(ImageStack& stack, std::vector& t_array, std::vector>& use_index_vect, @@ -251,7 +203,7 @@ std::vector StampCreator::get_coadded_stamps_gpu(ImageStack& stack, throw std::runtime_error("Non-GPU co-adds is not implemented."); #endif - // Copy the stamps into RawImages and do the filtering. + // Copy the stamps into RawImages. std::vector results(num_trajectories); std::vector current_pixels(stamp_ppi, 0.0); for (uint64_t t = 0; t < num_trajectories; ++t) { @@ -263,12 +215,7 @@ std::vector StampCreator::get_coadded_stamps_gpu(ImageStack& stack, Image tmp = Eigen::Map(current_pixels.data(), stamp_width, stamp_width); RawImage current_image = RawImage(tmp); - - if (params.do_filtering && filter_stamp(current_image, params)) { - results[t] = std::move(RawImage(1, 1, NO_DATA)); - } else { - results[t] = std::move(current_image); - } + results[t] = std::move(current_image); } return results; } @@ -351,8 +298,7 @@ static void stamp_creator_bindings(py::module& m) { pydocs::DOC_StampCreator_get_variance_weighted_stamp) .def_static("create_stamps", &sc::create_stamps, pydocs::DOC_StampCreator_create_stamps) .def_static("create_variance_stamps", &sc::create_variance_stamps, - pydocs::DOC_StampCreator_create_variance_stamps) - .def_static("filter_stamp", &sc::filter_stamp, pydocs::DOC_StampCreator_filter_stamp); + pydocs::DOC_StampCreator_create_variance_stamps); } #endif /* Py_PYTHON_H */ diff --git a/src/kbmod/search/stamp_creator.h b/src/kbmod/search/stamp_creator.h index 0cc924f8e..74dd82c4c 100644 --- a/src/kbmod/search/stamp_creator.h +++ b/src/kbmod/search/stamp_creator.h @@ -36,8 +36,7 @@ class StampCreator { // Compute a mean or summed stamp for each trajectory on the GPU or CPU. // The GPU implementation is slower for small numbers of trajectories (< 500), but performs - // relatively better as the number of trajectories increases. If filtering is applied then - // the code will return a 1x1 image with NO_DATA to represent each filtered image. + // relatively better as the number of trajectories increases. static std::vector get_coadded_stamps(ImageStack& stack, std::vector& t_array, std::vector >& use_index_vect, const StampParameters& params, bool use_gpu); @@ -50,9 +49,6 @@ class StampCreator { std::vector >& use_index_vect, const StampParameters& params); - // Function to do the actual stamp filtering. - static bool filter_stamp(const RawImage& img, const StampParameters& params); - // Function for generating variance stamps. All times are returned and NO_DATA values are preserved. static std::vector create_variance_stamps(ImageStack& stack, const Trajectory& trj, int radius, const std::vector& use_index); diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index ec6aa20ba..876cb5e72 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -30,11 +30,12 @@ def test_demo_defaults(self): # Load the WorkUnit. input_data = WorkUnit.from_fits(filename) + input_data.config.set("coadds", ["mean"]) rs = SearchRunner() keep = rs.run_search_from_work_unit(input_data) self.assertGreaterEqual(len(keep), 1) - self.assertEqual(keep["stamp"][0].shape, (21, 21)) + self.assertEqual(keep["coadd_mean"][0].shape, (21, 21)) @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") def test_demo_stamp_size(self): @@ -48,15 +49,15 @@ def test_demo_stamp_size(self): # Override the stamp settings of the configuration input_data.config.set("stamp_radius", 15) - input_data.config.set("mom_lims", [80.0, 80.0, 50.0, 20.0, 20.0]) input_data.config.set("save_all_stamps", True) + input_data.config.set("coadds", ["mean"]) rs = SearchRunner() keep = rs.run_search_from_work_unit(input_data) self.assertGreaterEqual(len(keep), 1) - self.assertIsNotNone(keep["stamp"][0]) - self.assertEqual(keep["stamp"][0].shape, (31, 31)) + self.assertIsNotNone(keep["coadd_mean"][0]) + self.assertEqual(keep["coadd_mean"][0].shape, (31, 31)) self.assertIsNotNone(keep["all_stamps"][0]) for s in keep["all_stamps"][0]: diff --git a/tests/test_raw_image.py b/tests/test_raw_image.py index 64f8b4589..d9a078871 100644 --- a/tests/test_raw_image.py +++ b/tests/test_raw_image.py @@ -203,75 +203,6 @@ def test_replace_masked_values(self): else: self.assertEqual(img2.get_pixel(row, col), 0.0) - def test_find_peak(self): - "Test RawImage find_peak" - img = RawImage(self.masked_array) - idx = img.find_peak(False) - self.assertEqual(idx.i, 5) - self.assertEqual(idx.j, 5) - - # We found the peak furthest to the center. - idx = img.find_peak(True) - self.assertEqual(idx.i, 3) - self.assertEqual(idx.j, 1) - - # We are okay when the data includes NaNs. - img.set_pixel(2, 3, math.nan) - img.set_pixel(3, 2, np.nan) - idx = img.find_peak(False) - self.assertEqual(idx.i, 5) - self.assertEqual(idx.j, 5) - - def test_find_central_moments(self): - """Test RawImage central moments.""" - img = RawImage(5, 5, value=0.1) - - # Try something mostly symmetric and centered. - img.set_pixel(2, 2, 10.0) - img.set_pixel(2, 1, 5.0) - img.set_pixel(1, 2, 5.0) - img.set_pixel(2, 3, 5.0) - img.set_pixel(3, 2, 5.0) - - img_mom = img.find_central_moments() - self.assertAlmostEqual(img_mom.m00, 1.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m01, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m10, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m11, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m02, 0.3322, delta=1e-4) - self.assertAlmostEqual(img_mom.m20, 0.3322, delta=1e-4) - - # Try something flat symmetric and centered. - img.set_all(2.0) - img_mom = img.find_central_moments() - - self.assertAlmostEqual(img_mom.m00, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m01, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m10, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m11, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m02, 0.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m20, 0.0, delta=1e-4) - - # Try something with a few non-symmetric peaks. - img.set_all(0.4) - img.set_pixel(2, 2, 5.0) - img.set_pixel(0, 1, 5.0) - img.set_pixel(3, 3, 10.0) - img.set_pixel(0, 3, 0.2) - img_mom = img.find_central_moments() - - self.assertAlmostEqual(img_mom.m00, 1.0, delta=1e-4) - self.assertAlmostEqual(img_mom.m01, 0.20339, delta=1e-4) - self.assertAlmostEqual(img_mom.m10, 0.03390, delta=1e-4) - self.assertAlmostEqual(img_mom.m11, 0.81356, delta=1e-4) - self.assertAlmostEqual(img_mom.m02, 1.01695, delta=1e-4) - self.assertAlmostEqual(img_mom.m20, 1.57627, delta=1e-4) - - # Check that nothing fails with NaNs. - img.set_pixel(2, 3, math.nan) - img.set_pixel(3, 2, np.nan) - img_mom = img.find_central_moments() - def convolve_psf_identity(self, device): psf_data = np.zeros((3, 3), dtype=np.single) psf_data[1, 1] = 1.0 diff --git a/tests/test_search.py b/tests/test_search.py index fda08ed5f..71a801b8b 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -83,16 +83,7 @@ def setUp(self): # Set the filtering parameters. self.params = StampParameters() self.params.radius = 5 - self.params.do_filtering = True self.params.stamp_type = StampType.STAMP_MEAN - self.params.center_thresh = 0.03 - self.params.peak_offset_x = 1.5 - self.params.peak_offset_y = 1.5 - self.params.m01_limit = 1.0 - self.params.m10_limit = 1.0 - self.params.m11_limit = 2.0 - self.params.m02_limit = 35.5 - self.params.m20_limit = 35.5 self.trj_gen = KBMODV1Search( self.velocity_steps, diff --git a/tests/test_stamp_creator.py b/tests/test_stamp_creator.py index 58a9567ab..53d107b1e 100644 --- a/tests/test_stamp_creator.py +++ b/tests/test_stamp_creator.py @@ -88,16 +88,7 @@ def setUp(self): # Set the filtering parameters. self.params = StampParameters() self.params.radius = 5 - self.params.do_filtering = True self.params.stamp_type = StampType.STAMP_MEAN - self.params.center_thresh = 0.03 - self.params.peak_offset_x = 1.5 - self.params.peak_offset_y = 1.5 - self.params.m01_limit = 1.0 - self.params.m10_limit = 1.0 - self.params.m11_limit = 2.0 - self.params.m02_limit = 35.5 - self.params.m20_limit = 35.5 # Create a second (smaller) fake data set to use in some of the tests. self.img_count2 = 10 @@ -367,53 +358,6 @@ def test_mean_stamps_no_data(self): # Check that we get the correct answer. self.assertAlmostEqual(pix_sum / pix_count, meanStamp.get_pixel(2, 2), delta=1e-5) - def test_filter_stamp(self): - stamp_width = 2 * self.params.radius + 1 - - # Test a stamp with nothing in it. - stamp = RawImage(stamp_width, stamp_width) - stamp.set_all(1.0) - self.assertTrue(StampCreator.filter_stamp(stamp, self.params)) - - # Test a stamp with a bright spot in the center. - stamp.set_pixel(5, 5, 100.0) - self.assertFalse(StampCreator.filter_stamp(stamp, self.params)) - - # A little noise around the pixel does not hurt as long as the shape is - # roughly Gaussian. - stamp.set_pixel(4, 5, 15.0) - stamp.set_pixel(5, 4, 10.0) - stamp.set_pixel(6, 5, 10.0) - stamp.set_pixel(5, 6, 20.0) - self.assertFalse(StampCreator.filter_stamp(stamp, self.params)) - - # A bright peak far from the center is bad. - stamp.set_pixel(1, 1, 500.0) - self.assertTrue(StampCreator.filter_stamp(stamp, self.params)) - stamp.set_pixel(1, 1, 1.0) - - # A non-Gaussian bright spot is also bad. Blur to the -x direction. - stamp.set_pixel(4, 5, 50.0) - stamp.set_pixel(3, 5, 50.0) - stamp.set_pixel(2, 5, 60.0) - stamp.set_pixel(4, 4, 45.0) - stamp.set_pixel(3, 4, 45.0) - stamp.set_pixel(2, 4, 55.0) - stamp.set_pixel(4, 6, 55.0) - stamp.set_pixel(3, 6, 55.0) - stamp.set_pixel(2, 6, 65.0) - self.assertTrue(StampCreator.filter_stamp(stamp, self.params)) - - # A very dim peak at the center is invalid. - stamp.set_all(1.0) - stamp.set_pixel(5, 5, 1.0001) - self.assertTrue(StampCreator.filter_stamp(stamp, self.params)) - - # A slightly offset peak of sufficient brightness is okay. - stamp.set_pixel(5, 5, 15.0) - stamp.set_pixel(4, 5, 20.0) - self.assertFalse(StampCreator.filter_stamp(stamp, self.params)) - def test_coadd_cpu_simple(self): # Create an image set with three images. imlist = [] @@ -451,7 +395,6 @@ def test_coadd_cpu_simple(self): # Basic Stamp parameters. params = StampParameters() params.radius = 1 - params.do_filtering = False # Test summed. params.stamp_type = StampType.STAMP_SUM @@ -524,7 +467,6 @@ def test_coadd_gpu_simple(self): # Basic Stamp parameters. params = StampParameters() params.radius = 1 - params.do_filtering = False # Test summed. params.stamp_type = StampType.STAMP_SUM @@ -550,7 +492,6 @@ def test_coadd_gpu_simple(self): def test_coadd_cpu(self): params = StampParameters() params.radius = 3 - params.do_filtering = False # Compute the stacked science (summed and mean) from a single Trajectory. params.stamp_type = StampType.STAMP_SUM @@ -605,7 +546,6 @@ def test_coadd_cpu(self): def test_coadd_gpu(self): params = StampParameters() params.radius = 3 - params.do_filtering = False # Compute the stacked science (summed and mean) from a single Trajectory. params.stamp_type = StampType.STAMP_SUM @@ -655,7 +595,6 @@ def test_coadd_gpu(self): def test_coadd_cpu_use_inds(self): params = StampParameters() params.radius = 1 - params.do_filtering = False params.stamp_type = StampType.STAMP_MEAN # Mark a few of the observations as "do not use" @@ -704,7 +643,6 @@ def test_coadd_cpu_use_inds(self): def test_coadd_gpu_use_inds(self): params = StampParameters() params.radius = 1 - params.do_filtering = False params.stamp_type = StampType.STAMP_MEAN # Mark a few of the observations as "do not use" @@ -749,83 +687,6 @@ def test_coadd_gpu_use_inds(self): self.assertAlmostEqual(sum_0 / count_0, meanStamps[0].get_pixel(stamp_y, stamp_x), delta=1e-3) self.assertAlmostEqual(sum_1 / count_1, meanStamps[1].get_pixel(stamp_y, stamp_x), delta=1e-3) - def test_coadd_filter_cpu(self): - # Create a second Trajectory that isn't any good. - trj2 = Trajectory(x=1, y=1, vx=0.0, vy=0.0) - - # Create a third Trajectory that is close to good, but offset. - trj3 = Trajectory( - x=self.trj.x + 2, - y=self.trj.y + 2, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Create a fourth Trajectory that is close enough - trj4 = Trajectory( - x=self.trj.x + 1, - y=self.trj.y + 1, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Compute the stacked science from a single Trajectory. - all_valid_vect = [(self.all_valid) for i in range(4)] - meanStamps = StampCreator.get_coadded_stamps( - self.stack, [self.trj, trj2, trj3, trj4], all_valid_vect, self.params, True - ) - - # The first and last are unfiltered - self.assertEqual(meanStamps[0].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[0].height, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].height, 2 * self.params.radius + 1) - - # The second and third are filtered. - self.assertEqual(meanStamps[1].width, 1) - self.assertEqual(meanStamps[1].height, 1) - self.assertEqual(meanStamps[2].width, 1) - self.assertEqual(meanStamps[2].height, 1) - - @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") - def test_coadd_filter_gpu(self): - # Create a second Trajectory that isn't any good. - trj2 = Trajectory(x=1, y=1, vx=0.0, vy=0.0) - - # Create a third Trajectory that is close to good, but offset. - trj3 = Trajectory( - x=self.trj.x + 2, - y=self.trj.y + 2, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Create a fourth Trajectory that is close enough - trj4 = Trajectory( - x=self.trj.x + 1, - y=self.trj.y + 1, - vx=self.trj.vx, - vy=self.trj.vy, - ) - - # Compute the stacked science from a single Trajectory. - all_valid_vect = [(self.all_valid) for i in range(4)] - meanStamps = StampCreator.get_coadded_stamps( - self.stack, [self.trj, trj2, trj3, trj4], all_valid_vect, self.params, True - ) - - # The first and last are unfiltered - self.assertEqual(meanStamps[0].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[0].height, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].width, 2 * self.params.radius + 1) - self.assertEqual(meanStamps[3].height, 2 * self.params.radius + 1) - - # The second and third are filtered. - self.assertEqual(meanStamps[1].width, 1) - self.assertEqual(meanStamps[1].height, 1) - self.assertEqual(meanStamps[2].width, 1) - self.assertEqual(meanStamps[2].height, 1) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_stamp_filters.py b/tests/test_stamp_filters.py index 9f3e71e1a..4ad82676d 100644 --- a/tests/test_stamp_filters.py +++ b/tests/test_stamp_filters.py @@ -28,10 +28,6 @@ def setUp(self): def test_extract_search_parameters_from_config(self): config_dict = { - "center_thresh": 0.05, - "do_stamp_filter": True, - "mom_lims": [50.0, 51.0, 1.0, 2.0, 3.0], - "peak_offset": [1.5, 3.5], "stamp_type": "median", "stamp_radius": 7, } @@ -40,15 +36,6 @@ def test_extract_search_parameters_from_config(self): params = extract_search_parameters_from_config(config) self.assertEqual(params.radius, 7) self.assertEqual(params.stamp_type, StampType.STAMP_MEDIAN) - self.assertEqual(params.do_filtering, True) - self.assertAlmostEqual(params.center_thresh, 0.05) - self.assertAlmostEqual(params.peak_offset_x, 1.5) - self.assertAlmostEqual(params.peak_offset_y, 3.5) - self.assertAlmostEqual(params.m20_limit, 50.0) - self.assertAlmostEqual(params.m02_limit, 51.0) - self.assertAlmostEqual(params.m11_limit, 1.0) - self.assertAlmostEqual(params.m10_limit, 2.0) - self.assertAlmostEqual(params.m01_limit, 3.0) # Test bad configurations config.set("stamp_radius", -1) @@ -59,16 +46,8 @@ def test_extract_search_parameters_from_config(self): self.assertRaises(ValueError, extract_search_parameters_from_config, config) config.set("stamp_type", "median") - config.set("peak_offset", [50.0]) - self.assertRaises(ValueError, extract_search_parameters_from_config, config) - config.set("peak_offset", [1.5, 3.5]) - - config.set("mom_lims", [50.0, 51.0, 1.0, 3.0]) - self.assertRaises(ValueError, extract_search_parameters_from_config, config) - config.set("mom_lims", [50.0, 51.0, 1.0, 2.0, 3.0]) - @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") - def test_get_coadds_and_filter_results(self): + def test_make_coadds(self): # Create trajectories to test: 0) known good, 1) completely wrong # 2) close to good, but offset], and 3) just close enough. trj_list = [ @@ -82,25 +61,18 @@ def test_get_coadds_and_filter_results(self): # Create the stamp parameters we need. config_dict = { - "center_thresh": 0.03, - "do_stamp_filter": True, - "mom_lims": [35.5, 35.5, 1.0, 1.0, 1.0], - "peak_offset": [1.5, 1.5], "stamp_type": "cpp_mean", "stamp_radius": 5, } config = SearchConfiguration.from_dict(config_dict) - # Do the filtering. - get_coadds_and_filter_results(keep, self.ds.stack, config, chunk_size=2) + # Make the stamps. + make_coadds(keep, self.ds.stack, config, chunk_size=2) - # The check that the correct indices and number of stamps are saved. + # We do not filter, so everything should be saved. self.assertTrue("stamp" in keep.colnames) - self.assertEqual(len(keep), 2) - self.assertEqual(keep["x"][0], self.trj.x) - self.assertEqual(keep["x"][1], self.trj.x + 1) + self.assertEqual(len(keep), 4) self.assertEqual(keep["stamp"][0].shape, (11, 11)) - self.assertEqual(keep["stamp"][1].shape, (11, 11)) @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") def test_get_coadds_and_filter_with_invalid(self): @@ -119,27 +91,19 @@ def test_get_coadds_and_filter_with_invalid(self): # Create the stamp parameters we need. config_dict = { - "center_thresh": 0.03, - "do_stamp_filter": True, - "mom_lims": [35.5, 35.5, 1.0, 1.0, 1.0], - "peak_offset": [1.5, 1.5], "stamp_type": "cpp_mean", "stamp_radius": 5, } config = SearchConfiguration.from_dict(config_dict) - # Do the filtering. - get_coadds_and_filter_results(keep, self.ds.stack, config, chunk_size=2) - - # The check that the correct indices and number of stamps are saved. + # Make the stamps and check that there were saved. + make_coadds(keep, self.ds.stack, config, chunk_size=2) self.assertTrue("stamp" in keep.colnames) - self.assertEqual(len(keep), 1) - self.assertEqual(keep["vx"][0], trj2.vx) - self.assertEqual(keep["vy"][0], trj2.vy) + self.assertEqual(len(keep), 2) # Test with empty results. keep2 = Results.from_trajectories([]) - get_coadds_and_filter_results(keep2, self.ds.stack, config, chunk_size=1000) + make_coadds(keep2, self.ds.stack, config, chunk_size=1000) self.assertTrue("stamp" in keep2.colnames) @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") diff --git a/tests/test_stamp_parity.py b/tests/test_stamp_parity.py index b4b580643..4a1c611bb 100644 --- a/tests/test_stamp_parity.py +++ b/tests/test_stamp_parity.py @@ -52,7 +52,6 @@ def test_coadd_gpu_parity(self): width = 2 * radius + 1 params = StampParameters() params.radius = radius - params.do_filtering = False results = [self.trj, self.trj] all_valid = [1] * self.img_count From ab3e8822ace39fa570d29c001a47323289a8f66a Mon Sep 17 00:00:00 2001 From: Jeremy Kubica <104161096+jeremykubica@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:55:07 -0500 Subject: [PATCH 2/4] Update the submodules --- include/eigen | 2 +- lib/pybind11 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/eigen b/include/eigen index c01ff4531..6d829e766 160000 --- a/include/eigen +++ b/include/eigen @@ -1 +1 @@ -Subproject commit c01ff45312582b2ea896ee307a49165ca4790332 +Subproject commit 6d829e766ff1b1ab867d93631163cbc63ed5798f diff --git a/lib/pybind11 b/lib/pybind11 index 741d86f2e..3efe9d4cb 160000 --- a/lib/pybind11 +++ b/lib/pybind11 @@ -1 +1 @@ -Subproject commit 741d86f2e3527b667ba85d273a5eea19a0978ef5 +Subproject commit 3efe9d4cb5d7314faee722205e560b8b932aed9e From af0497e5e85af0f6e3bca403a688e9c2e571e515 Mon Sep 17 00:00:00 2001 From: Jeremy Kubica <104161096+jeremykubica@users.noreply.github.com> Date: Mon, 3 Feb 2025 09:51:19 -0500 Subject: [PATCH 3/4] Keep generation of stamp column --- docs/source/user_manual/results_filtering.rst | 23 ------------------- docs/source/user_manual/search_params.rst | 23 ++----------------- src/kbmod/run_search.py | 13 ++++++++--- src/kbmod/search/stamp_creator.cpp | 8 +++---- tests/test_end_to_end.py | 6 +++++ tests/test_stamp_creator.py | 2 -- 6 files changed, 22 insertions(+), 53 deletions(-) diff --git a/docs/source/user_manual/results_filtering.rst b/docs/source/user_manual/results_filtering.rst index f2ea3b481..44589de66 100644 --- a/docs/source/user_manual/results_filtering.rst +++ b/docs/source/user_manual/results_filtering.rst @@ -21,29 +21,6 @@ Relevant light curve filtering parameters include: * ``sigmaG_lims`` - The percentiles for sigmaG filtering (default of [25, 75]). -Stamp Filtering ---------------- - -The stamp filtering stage is only applied if the ``do_stamp_filter`` parameter is set to True. This stage creates a single stamp representing the sum, mean, median, or variance weighted mean of pixel values for the stamps at each time step. The stamp type is defined by the ``stamp_type`` parameter and can take on values ``median``, ``mean``, ``sum``, or ``weighted`` (for variance weighted). All of the stamp types drop masked pixels from their computations. The mean and median sums are computed over only the valid time steps from the light curve filtering phase (dropping stamps with outlier fluxes). The sum coadd uses all the time steps regardless of the first phase of filtering. - -The stamps are filtered based on how closely the pixel values in the stamp image represent a Gaussian defined with the parameters: - -* ``center_thresh`` - The percentage of flux in the central pixel. For example setting this to 0.9 will require that the central pixel of the stamp has 90 percent of all the flux in the stamp. -* ``peak_offset`` - How far the brightest pixel is from the center of the stamp (in pixels). For example a peak offset of [2.0, 3.0] requires that the brightest pixel in the stamp is at most 2 pixels from the center in the x-dimension and 3-pixels from the center in the y-dimension. -* ``mom_lims`` - Compute the Gaussian moments of the image and compares them to the thresholds. - -Relevant stamp filtering parameters include: - - * ``center_thresh`` - The percentage of flux in the central pixel. - * ``chunk_size`` - The number of candidate trajectories to filter in a batch. Used to control memory usage. - * ``do_stamp_filter`` - A Boolean indicating whether to generate and filter stamps. - * ``peak_offset`` - A length 2 list indicating how far the peak is from the center of the stamp in each of the x and y dimensions. - * ``mom_lims`` - A length 5 list providing thresholds on the images moments. - * ``stamp_radius`` - The radius of the stamps to use. - -Note that stamps are only generated and output into files if ``do_stamp_filter`` is set to true. - - Clustering ---------- diff --git a/docs/source/user_manual/search_params.rst b/docs/source/user_manual/search_params.rst index 64cf5e37a..25690d39b 100644 --- a/docs/source/user_manual/search_params.rst +++ b/docs/source/user_manual/search_params.rst @@ -33,11 +33,6 @@ Configuration Parameters +------------------------+-----------------------------+----------------------------------------+ | **Parameter** | **Default Value** | **Interpretation** | +------------------------+-----------------------------+----------------------------------------+ -| ``center_thresh`` | 0.00 | The minimum fraction of total flux | -| | | within a stamp that must be contained | -| | | in the central pixel | -| | | (if ``do_stamp_filter=True``). | -+------------------------+-----------------------------+----------------------------------------+ | ``chunk_size`` | 500000 | The batch size to use when processing | | | | the results of the on-GPU search. | +------------------------+-----------------------------+----------------------------------------+ @@ -76,9 +71,6 @@ Configuration Parameters +------------------------+-----------------------------+----------------------------------------+ | ``do_mask`` | True | Apply the mask to the raw pixels. | +------------------------+-----------------------------+----------------------------------------+ -| ``do_stamp_filter`` | True | Apply post-search filtering on the | -| | | image stamps. | -+------------------------+-----------------------------+----------------------------------------+ | ``encode_num_bytes`` | -1 | The number of bytes to use to encode | | | | ``psi`` and ``phi`` images on GPU. By | | | | default a ``float`` encoding is used. | @@ -113,20 +105,10 @@ Configuration Parameters | | | computed likelihood above this | | | | threshold are rejected. | +------------------------+-----------------------------+----------------------------------------+ -| ``mom_lims`` | [35.5, 35.5, 2.0, 0.3, 0.3] | Thresholds for the moments of a | -| | | Gaussian fit to the flux, specified as | -| | | ``[xx, yy, xy, x, y]``. | -| | | If ``do_stamp_filter=True``. | -+------------------------+-----------------------------+----------------------------------------+ | ``num_obs`` | 10 | The minimum number of non-masked | | | | observations for the object to be | | | | accepted. | +------------------------+-----------------------------+----------------------------------------+ -| ``peak_offset`` | [2.0, 2.0] | How far, in pixels, the brightest pixel| -| | | in the stamp can be from the central | -| | | pixel in each direction ``[x,y]``. | -| | | If ``do_stamp_filter=True``). | -+------------------------+-----------------------------+----------------------------------------+ | ``psf_val`` | 1.4 | The value for the standard deviation of| | | | the point spread function (PSF). | +------------------------+-----------------------------+----------------------------------------+ @@ -148,9 +130,8 @@ Configuration Parameters | | | creating a stamp for stamp filtering | | | | (in pixels). | +------------------------+-----------------------------+----------------------------------------+ -| ``stamp_type`` | sum | The type of coadd to use during stamp | -| | | filtering (if ``do_stamp_filter=True``)| -| | | if: | +| ``stamp_type`` | sum | The type of coadd to use as the main | +| | | stamp: | | | | * ``sum`` - (default) Per pixel sum | | | | * ``median`` - Per pixel median | | | | * ``mean`` - Per pixel mean | diff --git a/src/kbmod/run_search.py b/src/kbmod/run_search.py index a94009464..e4ca80afb 100644 --- a/src/kbmod/run_search.py +++ b/src/kbmod/run_search.py @@ -9,7 +9,12 @@ from .configuration import SearchConfiguration from .filters.clustering_filters import apply_clustering from .filters.sigma_g_filter import apply_clipped_sigma_g, SigmaGClipping -from .filters.stamp_filters import append_all_stamps, append_coadds +from .filters.stamp_filters import ( + append_all_stamps, + append_coadds, + extract_search_parameters_from_config, + make_coadds, +) from .results import Results from .trajectory_generator import create_trajectory_generator @@ -250,12 +255,14 @@ def run_search( apply_clustering(keep, cluster_params) cluster_timer.stop() - # Generate additional coadded stamps without filtering. + # Generate coadded stamps without filtering -- both the "stamp" column + # as well as any additional coadds. + stamp_params = extract_search_parameters_from_config(config) + make_coadds(keep, stack, stamp_params, colname="stamp") if len(config["coadds"]) > 0: stack.copy_to_gpu() append_coadds(keep, stack, config["coadds"], config["stamp_radius"]) stack.clear_from_gpu() - logger.info(f"Found {len(keep)} potential trajectories.") # Extract all the stamps for all time steps and append them onto the result rows. if config["save_all_stamps"]: diff --git a/src/kbmod/search/stamp_creator.cpp b/src/kbmod/search/stamp_creator.cpp index 078c1d32f..b2aabc819 100644 --- a/src/kbmod/search/stamp_creator.cpp +++ b/src/kbmod/search/stamp_creator.cpp @@ -118,10 +118,10 @@ std::vector get_coadded_stamps_cpu(ImageStack& stack, std::vector StampCreator::get_coadded_stamps_gpu(ImageStack& stack, - std::vector& t_array, - std::vector>& use_index_vect, - const StampParameters& params) { +std::vector get_coadded_stamps_gpu(ImageStack& stack, + std::vector& t_array, + std::vector>& use_index_vect, + const StampParameters& params) { logging::Logger* rs_logger = logging::getLogger("kbmod.search.stamp_creator"); // Right now only limited stamp sizes are allowed. diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index a438d3d94..377232342 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -35,6 +35,9 @@ def test_demo_defaults(self): rs = SearchRunner() keep = rs.run_search_from_work_unit(input_data) self.assertGreaterEqual(len(keep), 1) + + # Check that we have both a stamp column and a coadd_mean column. + self.assertEqual(keep["stamp"][0].shape, (21, 21)) self.assertEqual(keep["coadd_mean"][0].shape, (21, 21)) @unittest.skipIf(not HAS_GPU, "Skipping test (no GPU detected)") @@ -56,6 +59,9 @@ def test_demo_stamp_size(self): keep = rs.run_search_from_work_unit(input_data) self.assertGreaterEqual(len(keep), 1) + self.assertIsNotNone(keep["stamp"][0]) + self.assertEqual(keep["stamp"][0].shape, (31, 31)) + self.assertIsNotNone(keep["coadd_mean"][0]) self.assertEqual(keep["coadd_mean"][0].shape, (31, 31)) diff --git a/tests/test_stamp_creator.py b/tests/test_stamp_creator.py index a748679f3..d5d92223b 100644 --- a/tests/test_stamp_creator.py +++ b/tests/test_stamp_creator.py @@ -25,7 +25,6 @@ get_variance_weighted_stamp, create_stamps, create_variance_stamps, - filter_stamp, ) @@ -607,7 +606,6 @@ def test_coadd_cpu_fallback(self): # Use a radius that is too large for the GPU. params = StampParameters() params.radius = 500 - params.do_filtering = False # Test that we get valid stamp values for all pixels. params.stamp_type = StampType.STAMP_MEAN From 8eef10630e798d9c7c16167b1162fd9b7e928ef8 Mon Sep 17 00:00:00 2001 From: Jeremy Kubica <104161096+jeremykubica@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:43:58 -0500 Subject: [PATCH 4/4] Update test_stamp_creator.py --- tests/test_stamp_creator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_stamp_creator.py b/tests/test_stamp_creator.py index a11f7400d..5e71f6205 100644 --- a/tests/test_stamp_creator.py +++ b/tests/test_stamp_creator.py @@ -610,7 +610,6 @@ def test_coadd_gpu_large(self): for radius in [10, 20, 30]: params = StampParameters() params.radius = radius - params.do_filtering = False # Test that we get valid stamp values for all pixels (not all zeros). params.stamp_type = StampType.STAMP_MEAN