diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index f2eeccc9e..c40efb556 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -1,739 +1,65 @@ """ -Module for WINTER data reduction +Module with blocks for WINTER data reduction """ # pylint: disable=duplicate-code -from mirar.catalog.kowalski import PS1, TMASS from mirar.downloader.get_test_data import get_test_data_dir from mirar.paths import ( BASE_NAME_KEY, DITHER_N_KEY, EXPTIME_KEY, - LATEST_SAVE_KEY, MAX_DITHER_KEY, OBSCLASS_KEY, - RAW_IMG_KEY, - SOURCE_HISTORY_KEY, - SOURCE_NAME_KEY, TARGET_KEY, - ZP_KEY, - base_output_dir, ) -from mirar.pipelines.winter.config import ( - prv_candidate_cols, - psfex_path, - scamp_config_path, - sextractor_astrometry_config, - sextractor_astromstats_config, - sextractor_candidate_config, - sextractor_photometry_config, - sextractor_photometry_psf_config, - sextractor_reference_psf_phot_config, - swarp_config_path, - winter_avro_schema_path, - winter_fritz_config, -) -from mirar.pipelines.winter.constants import NXSPLIT, NYSPLIT -from mirar.pipelines.winter.generator import ( - mask_stamps_around_bright_stars, - select_winter_flat_images, - winter_anet_sextractor_config_path_generator, - winter_astrometric_ref_catalog_generator, - winter_astrometric_ref_catalog_namer, - winter_astrometry_sextractor_catalog_purifier, - winter_astrostat_catalog_purifier, - winter_cal_requirements, - winter_candidate_annotator_filterer, - winter_candidate_avro_fields_calculator, - winter_candidate_quality_filterer, - winter_fourier_filtered_image_generator, - winter_history_deprecated_constraint, - winter_imsub_catalog_purifier, - winter_new_source_updater, - winter_photcal_color_columns_generator, - winter_photometric_catalog_generator, - winter_photometric_catalogs_purifier, - winter_photometric_ref_catalog_namer, - winter_reference_generator, - winter_reference_image_resampler_for_zogy, - winter_reference_psf_phot_sextractor, - winter_reference_psfex, - winter_reference_sextractor, - winter_skyportal_annotator, - winter_source_entry_updater, - winter_stackid_annotator, +from mirar.pipelines.winter.blocks_reduction import ( + cal_hunter, + csvlog, + dark_calibrate, + extract_all, + first_pass_flat_calibrate, + first_pass_stacking, + focus_subcoord, + fourier_filter, + load_astrometried, + load_calibrated, + load_raw, + load_unpacked, + mask, + mask_and_split, + photcal_and_export, + save_raw, + second_pass_astrometry, + second_pass_calibration, + second_pass_stack, + second_pass_validate_astrometry_export_and_filter, + select_subset, + stack_dithers, +) +from mirar.pipelines.winter.blocks_subtraction import ( + detect_candidates, + imsub, + process_candidates, + refbuild, ) from mirar.pipelines.winter.load_winter_image import ( - annotate_winter_subdet_headers, - get_raw_winter_mask, - load_stacked_winter_image, load_test_winter_image, load_winter_mef_image, - load_winter_stack, -) -from mirar.pipelines.winter.models import ( - DEFAULT_FIELD, - NAME_START, - SOURCE_PREFIX, - AstrometryStat, - Candidate, - Diff, - Exposure, - Raw, - Source, - Stack, -) -from mirar.pipelines.winter.validator import ( - masked_images_rejector, - poor_astrometric_quality_rejector, - winter_dark_oversubtraction_rejector, ) -from mirar.processors.astromatic import PSFex, Scamp -from mirar.processors.astromatic.sextractor.background_subtractor import ( - SextractorBkgSubtractor, -) -from mirar.processors.astromatic.sextractor.sextractor import Sextractor -from mirar.processors.astromatic.swarp.swarp import Swarp -from mirar.processors.astrometry.anet.anet_processor import AstrometryNet -from mirar.processors.astrometry.validate import AstrometryStatsWriter -from mirar.processors.avro import IPACAvroExporter -from mirar.processors.catalog_limiting_mag import CatalogLimitingMagnitudeCalculator +from mirar.pipelines.winter.models import DEFAULT_FIELD from mirar.processors.csvlog import CSVLog -from mirar.processors.dark import DarkCalibrator -from mirar.processors.database.database_inserter import ( - DatabaseImageBatchInserter, - DatabaseImageInserter, - DatabaseSourceInserter, -) -from mirar.processors.database.database_selector import ( - SelectSourcesWithMetadata, - SingleSpatialCrossmatchSource, -) -from mirar.processors.database.database_updater import ImageDatabaseMultiEntryUpdater -from mirar.processors.flat import FlatCalibrator -from mirar.processors.mask import ( # MaskAboveThreshold, - MaskDatasecPixels, - MaskPixelsFromFunction, -) -from mirar.processors.photcal import ZPWithColorTermCalculator -from mirar.processors.photcal.photcalibrator import PhotCalibrator from mirar.processors.photometry import AperturePhotometry, PSFPhotometry -from mirar.processors.reference import GetReferenceImage, ProcessReference -from mirar.processors.skyportal.skyportal_candidate import SkyportalCandidateUploader -from mirar.processors.sources import ( - CandidateNamer, - CustomSourceTableModifier, - ForcedPhotometryDetector, - SourceLoader, - SourceWriter, - ZOGYSourceDetector, -) -from mirar.processors.split import SUB_ID_KEY, SplitImage, SwarpImageSplitter +from mirar.processors.sources import ForcedPhotometryDetector from mirar.processors.utils import ( - CustomImageBatchModifier, - HeaderAnnotator, ImageBatcher, ImageDebatcher, ImageLoader, - ImagePlotter, - ImageRejector, ImageSaver, ImageSelector, MEFLoader, ) -from mirar.processors.utils.cal_hunter import CalHunter -from mirar.processors.xmatch import XMatch -from mirar.processors.zogy.reference_aligner import AlignReference -from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare - -build_test = [ - MEFLoader( - input_sub_dir="raw", - load_image=load_winter_mef_image, - ), - ImageBatcher("UTCTIME"), - CSVLog( - export_keys=[ - "UTCTIME", - "PROGNAME", - DITHER_N_KEY, - MAX_DITHER_KEY, - "FILTER", - EXPTIME_KEY, - OBSCLASS_KEY, - "BOARD_ID", - "BASENAME", - TARGET_KEY, - "RADEG", - "DECDEG", - "T_ROIC", - "FIELDID", - "FOCPOS", - ] - ), - ImageSelector( - ("BOARD_ID", "2"), - (OBSCLASS_KEY, ["dark", "science"]), - (EXPTIME_KEY, "120.0"), - ("filter", ["dark", "J"]), - ("FIELDID", ["3944", str(DEFAULT_FIELD)]), - ), - ImageSaver("testdata", output_dir=get_test_data_dir()), -] - -load_test = [ - ImageLoader( - input_img_dir=get_test_data_dir(), - input_sub_dir="raw", - load_image=load_test_winter_image, - ), - ImageBatcher("UTCTIME"), -] - -load_ref = [ - ImageLoader( - input_sub_dir="stack", - load_image=load_stacked_winter_image, - input_img_dir=base_output_dir, - ) -] - -refbuild = [ - GetReferenceImage(ref_image_generator=winter_reference_generator), - ImageSaver(output_dir_name="stacked_ref"), -] - -# Start for new WINTER blocks: - -# Loading - -load_raw = [ - MEFLoader( - input_sub_dir="raw", - load_image=load_winter_mef_image, - ), -] - -extract_all = [ - ImageBatcher("UTCTIME"), - DatabaseImageBatchInserter(db_table=Exposure, duplicate_protocol="replace"), - ImageSelector((OBSCLASS_KEY, ["dark", "science", "flat"])), -] - -csvlog = [ - CSVLog( - export_keys=[ - "UTCTIME", - "PROGNAME", - DITHER_N_KEY, - MAX_DITHER_KEY, - "FILTER", - EXPTIME_KEY, - OBSCLASS_KEY, - "BOARD_ID", - "BASENAME", - TARGET_KEY, - "RADEG", - "DECDEG", - "T_ROIC", - "FIELDID", - ] - ), -] - -select_split_subset = [ImageSelector(("SUBCOORD", "0_0"))] - -# Optional subset selection -BOARD_ID = 4 -select_subset = [ - ImageSelector( - ("BOARD_ID", str(BOARD_ID)), - ), -] - -select_ref = [ - ImageSelector( - ("FIELDID", str(3944)), - ("BOARD_ID", str(BOARD_ID)), - ), - ImageDebatcher(), - ImageBatcher("STACKID"), -] - -# mask -mask = [ - ImageBatcher(BASE_NAME_KEY), - # MaskAboveThreshold(threshold=40000.0), - MaskDatasecPixels(), - MaskPixelsFromFunction(mask_function=get_raw_winter_mask), -] - -# Split -split = [ - SplitImage(n_x=NXSPLIT, n_y=NYSPLIT), - CustomImageBatchModifier(annotate_winter_subdet_headers), -] - -mask_and_split = mask + split - -# Save raw images - -save_raw = [ - ImageSaver(output_dir_name="raw_unpacked", write_mask=False), - DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace"), - ImageDebatcher(), - ImageBatcher(["BOARD_ID", "FILTER", "EXPTIME", TARGET_KEY, "SUBCOORD"]), - CustomImageBatchModifier(winter_stackid_annotator), - ImageSaver(output_dir_name="raw_unpacked", write_mask=False), - HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), - ImageRejector(("BOARD_ID", "0")), -] - -# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -# Various processing steps - -# Load from unpacked dir - -load_unpacked = [ - ImageLoader(input_sub_dir="raw_unpacked", input_img_dir=base_output_dir), - ImageDebatcher(), - ImageBatcher("UTCTIME"), - CSVLog( - export_keys=[ - "UTCTIME", - "PROGNAME", - DITHER_N_KEY, - MAX_DITHER_KEY, - "FILTER", - EXPTIME_KEY, - OBSCLASS_KEY, - "BOARD_ID", - "BASENAME", - TARGET_KEY, - "RADEG", - "DECDEG", - "T_ROIC", - "FIELDID", - "MEDCOUNT", - ] - ), - DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace"), - ImageRejector(("BOARD_ID", "0")), -] - -# -cal_hunter = [ - CalHunter(load_image=load_winter_mef_image, requirements=winter_cal_requirements) -] -# Detrend blocks - -dark_calibrate = [ - ImageDebatcher(), - ImageBatcher( - ["BOARD_ID", EXPTIME_KEY, "SUBCOORD", "GAINCOLT", "GAINCOLB", "GAINROW"] - ), - DarkCalibrator( - cache_sub_dir="calibration_darks", - cache_image_name_header_keys=[EXPTIME_KEY, "BOARD_ID"], - ), - ImageSelector((OBSCLASS_KEY, ["science", "flat"])), - ImageDebatcher(), - ImageBatcher(["BOARD_ID", "UTCTIME", "SUBCOORD"]), - ImageSaver(output_dir_name="darkcal"), - CustomImageBatchModifier(winter_dark_oversubtraction_rejector), -] - -flat_calibrate = [ - ImageDebatcher(), - ImageBatcher( - [ - "BOARD_ID", - "FILTER", - "SUBCOORD", - "GAINCOLT", - "GAINCOLB", - "GAINROW", - ] - ), - FlatCalibrator( - cache_sub_dir="calibration_flats", - select_flat_images=select_winter_flat_images, - cache_image_name_header_keys=["FILTER", "BOARD_ID"], - ), - ImageSaver(output_dir_name="skyflatcal"), - ImageDebatcher(), - ImageBatcher(["BOARD_ID", "UTCTIME", "SUBCOORD"]), - Sextractor( - **sextractor_astrometry_config, - write_regions_bool=True, - output_sub_dir="skysub", - checkimage_type=["-BACKGROUND"], - ), - SextractorBkgSubtractor(), - ImageSaver(output_dir_name="skysub"), -] - -load_calibrated = [ - ImageLoader(input_sub_dir="skysub", input_img_dir=base_output_dir), - ImageBatcher(["UTCTIME", "BOARD_ID"]), -] - -fourier_filter = [ - CustomImageBatchModifier(winter_fourier_filtered_image_generator), -] - -astrometry = [ - ImageDebatcher(), - ImageBatcher(["UTCTIME", "BOARD_ID", "SUBCOORD"]), - AstrometryNet( - output_sub_dir="anet", - scale_bounds=[1.0, 1.3], - scale_units="app", - use_sextractor=True, - parity="neg", - search_radius_deg=5.0, - sextractor_config_path=winter_anet_sextractor_config_path_generator, - use_weight=True, - timeout=120, - cache=True, - ), - ImageSaver(output_dir_name="post_anet"), - ImageDebatcher(), - ImageBatcher( - [TARGET_KEY, "FILTER", EXPTIME_KEY, "BOARD_ID", "SUBCOORD", "DITHGRP"] - ), - Sextractor( - **sextractor_astrometry_config, - write_regions_bool=True, - output_sub_dir="scamp", - catalog_purifier=winter_astrometry_sextractor_catalog_purifier, - ), - CustomImageBatchModifier(winter_astrometric_ref_catalog_namer), - Scamp( - scamp_config_path=scamp_config_path, - ref_catalog_generator=winter_astrometric_ref_catalog_generator, - copy_scamp_header_to_image=True, - cache=True, - ), - ImageSaver(output_dir_name="post_scamp"), -] - -validate_astrometry = [ - ImageDebatcher(), - ImageBatcher(["UTCTIME", "BOARD_ID", "SUBCOORD"]), - Sextractor( - **sextractor_astromstats_config, - write_regions_bool=True, - output_sub_dir="astrostats", - ), - AstrometryStatsWriter( - ref_catalog_generator=winter_astrometric_ref_catalog_generator, - image_catalog_purifier=winter_astrostat_catalog_purifier, - write_regions=True, - cache=False, - crossmatch_radius_arcsec=5.0, - ), - DatabaseImageInserter(db_table=AstrometryStat, duplicate_protocol="ignore"), - CustomImageBatchModifier(poor_astrometric_quality_rejector), -] - -stack_dithers = [ - ImageDebatcher(), - ImageBatcher(["BOARD_ID", "FILTER", "EXPTIME", TARGET_KEY, "SUBCOORD"]), - Swarp( - swarp_config_path=swarp_config_path, - calculate_dims_in_swarp=True, - include_scamp=True, - subtract_bkg=False, - cache=False, - center_type="ALL", - temp_output_sub_dir="stacks_weights", - header_keys_to_combine=["RAWID"], - ), - ImageSaver(output_dir_name="stack"), -] - -photcal_and_export = [ - ImageDebatcher(), - ImageBatcher(["BOARD_ID", "FILTER", TARGET_KEY, "SUBCOORD"]), - HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), - CustomImageBatchModifier(masked_images_rejector), - Sextractor( - **sextractor_photometry_config, - output_sub_dir="stack_psf", - checkimage_type="BACKGROUND_RMS", - ), - PSFex(config_path=psfex_path, output_sub_dir="phot", norm_fits=True), - Sextractor( - **sextractor_photometry_psf_config, - output_sub_dir="phot", - checkimage_type="BACKGROUND_RMS", - use_psfex=True, - ), - CustomImageBatchModifier(winter_photometric_ref_catalog_namer), - PhotCalibrator( - ref_catalog_generator=winter_photometric_catalog_generator, - catalogs_purifier=winter_photometric_catalogs_purifier, - temp_output_sub_dir="phot", - write_regions=True, - cache=True, - zp_calculator=ZPWithColorTermCalculator( - color_colnames_guess_generator=winter_photcal_color_columns_generator, - reject_outliers=True, - solver="curve_fit", - ), - zp_column_name="MAG_AUTO", - ), - CatalogLimitingMagnitudeCalculator( - sextractor_mag_key_name="MAG_AUTO", write_regions=True - ), - AstrometryStatsWriter( - ref_catalog_generator=winter_astrometric_ref_catalog_generator, - image_catalog_purifier=winter_astrostat_catalog_purifier, - write_regions=True, - cache=False, - crossmatch_radius_arcsec=5.0, - ), - ImageSaver(output_dir_name="final"), - DatabaseImageInserter(db_table=Stack, duplicate_protocol="replace"), - ImageDatabaseMultiEntryUpdater( - sequence_key="rawid", - db_table=Raw, - db_alter_columns="ustackid", - ), - ImagePlotter( - output_sub_dir="final_stacks_plots", - annotate_fields=[ - BASE_NAME_KEY, - "COADDS", - TARGET_KEY, - "CRVAL1", - "CRVAL2", - "FILTER", - "ZP", - "ZPSTD", - ], - ), -] - -# Image subtraction - -load_final_stack = [ - ImageLoader( - input_sub_dir="final", - input_img_dir=base_output_dir, - load_image=load_winter_stack, - ), - DatabaseImageInserter(db_table=Stack, duplicate_protocol="ignore"), -] - -plot_stack = [ - ImageDebatcher(), - ImageBatcher([TARGET_KEY, "BOARD_ID"]), - ImagePlotter( - output_sub_dir="final_stacks_plots", - annotate_fields=[ - BASE_NAME_KEY, - "COADDS", - TARGET_KEY, - "CRVAL1", - "CRVAL2", - "FILTER", - "ZP", - "ZPSTD", - ], - ), -] - -split_stack = [ - ImageDebatcher(), - ImageBatcher(["BOARD_ID", "FILTER", TARGET_KEY, "SUBCOORD", "STACKID"]), - SwarpImageSplitter(swarp_config_path=swarp_config_path, n_x=2, n_y=1), - ImageSaver(output_dir_name="split_stacks"), -] - -imsub = [ - ImageDebatcher(), - ImageBatcher(["BOARD_ID", "FILTER", TARGET_KEY, "SUBCOORD", "STACKID"]), - HeaderAnnotator(input_keys=[SUB_ID_KEY], output_key="SUBDETID"), - ProcessReference( - ref_image_generator=winter_reference_generator, - swarp_resampler=winter_reference_image_resampler_for_zogy, - sextractor=winter_reference_sextractor, - ref_psfex=winter_reference_psfex, - phot_sextractor=winter_reference_psf_phot_sextractor, - ), - Sextractor( - **sextractor_reference_psf_phot_config, - output_sub_dir="subtract", - cache=False, - use_psfex=True, - ), - PSFex(config_path=psfex_path, output_sub_dir="subtract", norm_fits=True), - AlignReference( - order=1, - sextractor=winter_reference_sextractor, - psfex=winter_reference_psfex, - phot_sextractor=winter_reference_psf_phot_sextractor, - catalog_purifier=winter_imsub_catalog_purifier, - ), - MaskPixelsFromFunction(mask_function=mask_stamps_around_bright_stars), - ImageSaver(output_dir_name="presubtract"), - ZOGYPrepare( - output_sub_dir="subtract", - sci_zp_header_key="ZP_AUTO", - ref_zp_header_key=ZP_KEY, - catalog_purifier=winter_imsub_catalog_purifier, - x_key="XMODEL_IMAGE", - y_key="YMODEL_IMAGE", - flux_key="FLUX_POINTSOURCE", - ), - # ImageSaver(output_dir_name="prezogy"), - ZOGY( - output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO", ref_zp_header_key=ZP_KEY - ), - ImageSaver(output_dir_name="diffs"), - DatabaseImageInserter(db_table=Diff, duplicate_protocol="replace"), - ImageSaver(output_dir_name="subtract"), -] - -load_sub = [ - ImageLoader(input_sub_dir="subtract"), -] -detect_candidates = [ - ZOGYSourceDetector( - output_sub_dir="subtract", - **sextractor_candidate_config, - write_regions=True, - detect_negative_sources=True, - ), - PSFPhotometry(phot_cutout_half_size=10), - AperturePhotometry( - temp_output_sub_dir="aper_photometry", - aper_diameters=[8, 16], - phot_cutout_half_size=50, - bkg_in_diameters=[25, 25], - bkg_out_diameters=[40, 40], - col_suffix_list=["", "big"], - ), - CustomSourceTableModifier(winter_candidate_annotator_filterer), - SourceWriter(output_dir_name="candidates"), -] -# -# candidate_colnames = get_column_names_from_schema(winter_candidate_config) - -load_sources = [ - SourceLoader(input_dir_name="candidates"), -] - -crossmatch_candidates = [ - XMatch(catalog=TMASS(num_sources=3, search_radius_arcmin=0.5)), - XMatch(catalog=PS1(num_sources=3, search_radius_arcmin=0.5)), - SourceWriter(output_dir_name="kowalski"), - CustomSourceTableModifier( - modifier_function=winter_candidate_avro_fields_calculator - ), -] - -select_history = [ - SelectSourcesWithMetadata( - db_query_columns=["sourceid"], - db_table=Candidate, - db_output_columns=prv_candidate_cols + [SOURCE_NAME_KEY], - base_output_column=SOURCE_HISTORY_KEY, - additional_query_constraints=winter_history_deprecated_constraint, - ), -] - -name_candidates = ( - [ - # Check if the source is already in the source table - SingleSpatialCrossmatchSource( - db_table=Source, - db_output_columns=["sourceid", SOURCE_NAME_KEY], - crossmatch_radius_arcsec=2.0, - ra_field_name="average_ra", - dec_field_name="average_dec", - ), - # Assign names to the new sources - CandidateNamer( - db_table=Source, - base_name=SOURCE_PREFIX, - name_start=NAME_START, - db_name_field=SOURCE_NAME_KEY, - ), - # Add the new sources to the source table - CustomSourceTableModifier(modifier_function=winter_new_source_updater), - DatabaseSourceInserter( - db_table=Source, - duplicate_protocol="ignore", - ), - # Get all candidates associated with source - ] - + select_history - + [ - # Update average ra and dec for source - CustomSourceTableModifier(modifier_function=winter_source_entry_updater), - # Update sources in the source table - DatabaseSourceInserter( - db_table=Source, - duplicate_protocol="replace", - ), - # Add candidates in the candidate table - DatabaseSourceInserter( - db_table=Candidate, - duplicate_protocol="fail", - ), - SourceWriter(output_dir_name="preavro"), - ] -) - -avro_export = [ - IPACAvroExporter( - topic_prefix="winter", - base_name="WNTR", - broadcast=False, - avro_schema_path=winter_avro_schema_path, - ), - CustomSourceTableModifier(modifier_function=winter_candidate_quality_filterer), - SourceWriter(output_dir_name="preskyportal"), -] - -process_candidates = crossmatch_candidates + name_candidates + avro_export - -load_skyportal = [SourceLoader(input_dir_name="preskyportal")] - -send_to_skyportal = [ - CustomSourceTableModifier(modifier_function=winter_skyportal_annotator), - SkyportalCandidateUploader(**winter_fritz_config), -] - -# To make a mosaic by stacking all boards -stack_boards = [ - ImageBatcher([TARGET_KEY]), - Swarp( - swarp_config_path=swarp_config_path, - calculate_dims_in_swarp=True, - include_scamp=False, - subtract_bkg=False, - cache=False, - center_type="ALL", - temp_output_sub_dir="mosaic_weights", - ), - ImageSaver(output_dir_name="mosaic"), -] - -mosaic = load_final_stack + stack_boards - -# To make cals for focusing -focus_subcoord = [ - HeaderAnnotator(input_keys=["BOARD_ID"], output_key="SUBCOORD"), - HeaderAnnotator(input_keys=["BOARD_ID"], output_key="SUBDETID"), -] # Combinations of different blocks, to be used in configurations -process_and_stack = astrometry + validate_astrometry + stack_dithers - unpack_subset = ( load_raw + cal_hunter @@ -752,47 +78,55 @@ unpack_all_no_calhunter = load_raw + extract_all + csvlog + mask_and_split + save_raw +first_pass_processing = ( + load_unpacked + + dark_calibrate + + first_pass_flat_calibrate + + fourier_filter + + first_pass_stacking +) + +second_pass_processing = ( + load_astrometried + stack_dithers + second_pass_calibration + second_pass_stack +) + +detrend_unpacked = load_unpacked + dark_calibrate + first_pass_flat_calibrate + +perform_astrometry = load_calibrated + fourier_filter + second_pass_astrometry + full_reduction = ( dark_calibrate - + flat_calibrate + + first_pass_flat_calibrate + fourier_filter - + process_and_stack + + first_pass_stacking + + second_pass_calibration + + fourier_filter + + second_pass_stack + photcal_and_export ) -photcal_stacks = [ - ImageLoader( - input_sub_dir="stack", - input_img_dir=base_output_dir, - load_image=load_winter_stack, - ), -] + photcal_and_export - reduce_unpacked = load_unpacked + full_reduction reduce = unpack_all + full_reduction reduce_no_calhunter = unpack_all_no_calhunter + full_reduction -reftest = ( - unpack_subset - + dark_calibrate - + flat_calibrate - + process_and_stack - + select_ref - + refbuild +process_and_stack = ( + second_pass_astrometry + + second_pass_validate_astrometry_export_and_filter + + stack_dithers + + [ + ImageSaver(output_dir_name="stack"), + ] ) -detrend_unpacked = load_unpacked + dark_calibrate + flat_calibrate - -only_ref = load_ref + select_ref + refbuild - realtime = extract_all + mask_and_split + save_raw + full_reduction -candidates = detect_candidates + process_candidates - full = realtime + imsub +candidates = detect_candidates + process_candidates + +# Other miscellaneous blocks focus_cals = ( load_raw + extract_all @@ -800,7 +134,7 @@ + focus_subcoord + csvlog + dark_calibrate - + flat_calibrate + + first_pass_flat_calibrate ) stack_forced_photometry = [ @@ -828,5 +162,65 @@ PSFPhotometry(), ] -perform_astrometry = load_calibrated + fourier_filter + astrometry -# + validate_astrometry +# Blocks for testing +build_test = [ + MEFLoader( + input_sub_dir="raw", + load_image=load_winter_mef_image, + ), + ImageBatcher("UTCTIME"), + CSVLog( + export_keys=[ + "UTCTIME", + "PROGNAME", + DITHER_N_KEY, + MAX_DITHER_KEY, + "FILTER", + EXPTIME_KEY, + OBSCLASS_KEY, + "BOARD_ID", + "BASENAME", + TARGET_KEY, + "RADEG", + "DECDEG", + "T_ROIC", + "FIELDID", + "FOCPOS", + ] + ), + ImageSelector( + ("BOARD_ID", "2"), + (OBSCLASS_KEY, ["dark", "science"]), + (EXPTIME_KEY, "120.0"), + ("filter", ["dark", "J"]), + ("FIELDID", ["3944", str(DEFAULT_FIELD)]), + ), + ImageSaver("testdata", output_dir=get_test_data_dir()), +] + +load_test = [ + ImageLoader( + input_img_dir=get_test_data_dir(), + input_sub_dir="raw", + load_image=load_test_winter_image, + ), + ImageBatcher("UTCTIME"), +] + +select_for_test_ref = [ + ImageSelector( + ("FIELDID", str(3944)), + ("BOARD_ID", str(4)), + ), + ImageDebatcher(), + ImageBatcher("STACKID"), +] + +reftest = ( + unpack_subset + + dark_calibrate + + first_pass_flat_calibrate + + process_and_stack + + select_for_test_ref + + refbuild +) diff --git a/mirar/pipelines/winter/blocks_reduction.py b/mirar/pipelines/winter/blocks_reduction.py new file mode 100644 index 000000000..0a0a450b3 --- /dev/null +++ b/mirar/pipelines/winter/blocks_reduction.py @@ -0,0 +1,576 @@ +""" +Module with blocks for WINTER image reduction and calibration +""" + +# pylint: disable=duplicate-code +from mirar.paths import ( + BASE_NAME_KEY, + DITHER_N_KEY, + EXPTIME_KEY, + FITS_MASK_KEY, + LATEST_SAVE_KEY, + MAX_DITHER_KEY, + OBSCLASS_KEY, + RAW_IMG_KEY, + TARGET_KEY, + base_output_dir, +) +from mirar.pipelines.winter.config import ( + psfex_path, + scamp_config_path, + sextractor_astrometry_config, + sextractor_astromstats_config, + sextractor_photometry_config, + sextractor_photometry_psf_config, + swarp_config_path, +) +from mirar.pipelines.winter.constants import NXSPLIT, NYSPLIT +from mirar.pipelines.winter.generator import ( + select_winter_flat_images, + winter_anet_sextractor_config_path_generator, + winter_astrometric_ref_catalog_generator, + winter_astrometric_ref_catalog_namer, + winter_astrometry_sextractor_catalog_purifier, + winter_astrostat_catalog_purifier, + winter_cal_requirements, + winter_fourier_filtered_image_generator, + winter_photcal_color_columns_generator, + winter_photometric_catalog_generator, + winter_photometric_catalogs_purifier, + winter_photometric_ref_catalog_namer, + winter_stackid_annotator, +) +from mirar.pipelines.winter.load_winter_image import ( + annotate_winter_subdet_headers, + get_raw_winter_mask, + load_winter_mef_image, + load_winter_stack, +) +from mirar.pipelines.winter.models import ( + AstrometryStat, + Exposure, + FirstPassAstrometryStat, + Raw, + Stack, +) +from mirar.pipelines.winter.validator import ( + masked_images_rejector, + poor_astrometric_quality_rejector, + winter_dark_oversubtraction_rejector, +) +from mirar.processors.astromatic import PSFex, Scamp +from mirar.processors.astromatic.scamp.scamp import SCAMP_HEADER_KEY +from mirar.processors.astromatic.sextractor.background_subtractor import ( + SextractorBkgSubtractor, +) +from mirar.processors.astromatic.sextractor.sextractor import ( + Sextractor, + sextractor_checkimg_map, +) +from mirar.processors.astromatic.swarp import ReloadSwarpComponentImages +from mirar.processors.astromatic.swarp.swarp import Swarp +from mirar.processors.astrometry.anet.anet_processor import AstrometryNet +from mirar.processors.astrometry.utils import AstrometryFromFile +from mirar.processors.astrometry.validate import AstrometryStatsWriter +from mirar.processors.catalog_limiting_mag import CatalogLimitingMagnitudeCalculator +from mirar.processors.csvlog import CSVLog +from mirar.processors.dark import DarkCalibrator +from mirar.processors.database.database_inserter import ( + DatabaseImageBatchInserter, + DatabaseImageInserter, +) +from mirar.processors.database.database_updater import ImageDatabaseMultiEntryUpdater +from mirar.processors.flat import FlatCalibrator, SkyFlatCalibrator +from mirar.processors.mask import ( # MaskAboveThreshold, + MaskDatasecPixels, + MaskPixelsFromFunction, + MaskPixelsFromPathInverted, + MaskPixelsFromWCS, + WriteMaskedCoordsToFile, +) +from mirar.processors.photcal import ZPWithColorTermCalculator +from mirar.processors.photcal.photcalibrator import PhotCalibrator +from mirar.processors.split import SplitImage +from mirar.processors.utils import ( + CustomImageBatchModifier, + HeaderAnnotator, + ImageBatcher, + ImageDebatcher, + ImageLoader, + ImagePlotter, + ImageRejector, + ImageSaver, + ImageSelector, + MEFLoader, +) +from mirar.processors.utils.cal_hunter import CalHunter +from mirar.processors.utils.image_loader import LoadImageFromHeader + +# Start for new WINTER blocks: + +load_raw = [ + MEFLoader( + input_sub_dir="raw", + load_image=load_winter_mef_image, + ), +] + +extract_all = [ + ImageBatcher("UTCTIME"), + DatabaseImageBatchInserter(db_table=Exposure, duplicate_protocol="replace"), + ImageSelector((OBSCLASS_KEY, ["dark", "science", "flat"])), +] + +csvlog = [ + CSVLog( + export_keys=[ + "UTCTIME", + "PROGNAME", + DITHER_N_KEY, + MAX_DITHER_KEY, + "FILTER", + EXPTIME_KEY, + OBSCLASS_KEY, + "BOARD_ID", + "BASENAME", + TARGET_KEY, + "RADEG", + "DECDEG", + "T_ROIC", + "FIELDID", + ] + ), +] + +select_split_subset = [ImageSelector(("SUBCOORD", "0_0"))] + +# Optional subset selection +BOARD_ID = 4 +select_subset = [ + ImageSelector( + ("BOARD_ID", str(BOARD_ID)), + ), +] + +# mask +mask = [ + ImageBatcher(BASE_NAME_KEY), + # MaskAboveThreshold(threshold=40000.0), + MaskDatasecPixels(), + MaskPixelsFromFunction(mask_function=get_raw_winter_mask), +] + +# Split +split = [ + SplitImage(n_x=NXSPLIT, n_y=NYSPLIT), + CustomImageBatchModifier(annotate_winter_subdet_headers), +] + +mask_and_split = mask + split + +# Save raw images + +save_raw = [ + ImageSaver(output_dir_name="raw_unpacked", write_mask=False), + DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace"), + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", "EXPTIME", TARGET_KEY, "SUBCOORD"]), + CustomImageBatchModifier(winter_stackid_annotator), + ImageSaver(output_dir_name="raw_unpacked", write_mask=False), + HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), + ImageRejector(("BOARD_ID", "0")), +] + +# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# Various processing steps + +# Load from unpacked dir + +load_unpacked = [ + ImageLoader(input_sub_dir="raw_unpacked", input_img_dir=base_output_dir), + HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), + ImageRejector(("BOARD_ID", "0")), + ImageDebatcher(), + ImageBatcher("UTCTIME"), + CSVLog( + export_keys=[ + "UTCTIME", + "PROGNAME", + DITHER_N_KEY, + MAX_DITHER_KEY, + "FILTER", + EXPTIME_KEY, + OBSCLASS_KEY, + "BOARD_ID", + "BASENAME", + TARGET_KEY, + "RADEG", + "DECDEG", + "T_ROIC", + "FIELDID", + "MEDCOUNT", + ] + ), + DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace"), + ImageRejector(("BOARD_ID", "0")), +] + +# Calibration hunter +cal_hunter = [ + CalHunter(load_image=load_winter_mef_image, requirements=winter_cal_requirements) +] + +# Detrend blocks +dark_calibrate = [ + ImageDebatcher(), + ImageBatcher( + ["BOARD_ID", EXPTIME_KEY, "SUBCOORD", "GAINCOLT", "GAINCOLB", "GAINROW"] + ), + DarkCalibrator( + cache_sub_dir="calibration_darks", + cache_image_name_header_keys=[EXPTIME_KEY, "BOARD_ID"], + ), + ImageSelector((OBSCLASS_KEY, ["science", "flat"])), + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "UTCTIME", "SUBCOORD"]), + ImageSaver(output_dir_name="darkcal"), + HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), + CustomImageBatchModifier(winter_dark_oversubtraction_rejector), +] + +# First pass flat calibration +first_pass_flat_calibrate = [ + ImageDebatcher(), + ImageBatcher( + [ + "BOARD_ID", + "FILTER", + "SUBCOORD", + "GAINCOLT", + "GAINCOLB", + "GAINROW", + TARGET_KEY, + ] + ), + FlatCalibrator( + cache_sub_dir="fp_flats", + select_flat_images=select_winter_flat_images, + cache_image_name_header_keys=["FILTER", "BOARD_ID", TARGET_KEY], + ), + ImageSaver(output_dir_name="skyflatcal"), + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "UTCTIME", "SUBCOORD"]), + Sextractor( + **sextractor_astrometry_config, + write_regions_bool=True, + output_sub_dir="fp_skysub", + checkimage_type=["-BACKGROUND"], + ), + SextractorBkgSubtractor(), + ImageSaver(output_dir_name="fp_skysub"), +] + +# Fourier filtering +fourier_filter = [ + CustomImageBatchModifier(winter_fourier_filtered_image_generator), +] + +# Various astrometry-related blocks +astrometry_net = [ + ImageDebatcher(), + ImageBatcher(["UTCTIME", "BOARD_ID", "SUBCOORD"]), + AstrometryNet( + output_sub_dir="anet", + scale_bounds=[1.0, 1.3], + scale_units="app", + use_sextractor=True, + parity="neg", + search_radius_deg=5.0, + sextractor_config_path=winter_anet_sextractor_config_path_generator, + use_weight=True, + timeout=120, + cache=True, + ), +] + +astrometry_scamp = [ + ImageDebatcher(), + ImageBatcher( + [TARGET_KEY, "FILTER", EXPTIME_KEY, "BOARD_ID", "SUBCOORD", "DITHGRP"] + ), + Sextractor( + **sextractor_astrometry_config, + write_regions_bool=True, + output_sub_dir="scamp", + catalog_purifier=winter_astrometry_sextractor_catalog_purifier, + ), + CustomImageBatchModifier(winter_astrometric_ref_catalog_namer), + Scamp( + scamp_config_path=scamp_config_path, + ref_catalog_generator=winter_astrometric_ref_catalog_generator, + copy_scamp_header_to_image=True, + cache=True, + ), +] + +validate_astrometry = [ + ImageDebatcher(), + ImageBatcher(["UTCTIME", "BOARD_ID", "SUBCOORD"]), + Sextractor( + **sextractor_astromstats_config, + write_regions_bool=True, + output_sub_dir="astrostats", + ), + AstrometryStatsWriter( + ref_catalog_generator=winter_astrometric_ref_catalog_generator, + image_catalog_purifier=winter_astrostat_catalog_purifier, + write_regions=True, + cache=False, + crossmatch_radius_arcsec=5.0, + ), +] + +first_pass_validate_astrometry_export_and_filter = validate_astrometry + [ + DatabaseImageInserter( + db_table=FirstPassAstrometryStat, duplicate_protocol="replace" + ), + CustomImageBatchModifier(poor_astrometric_quality_rejector), +] + +second_pass_validate_astrometry_export_and_filter = validate_astrometry + [ + DatabaseImageInserter(db_table=AstrometryStat, duplicate_protocol="replace"), + CustomImageBatchModifier(poor_astrometric_quality_rejector), +] + +load_calibrated = [ + ImageLoader(input_sub_dir="skysub", input_img_dir=base_output_dir), + ImageBatcher(["UTCTIME", "BOARD_ID"]), +] + +# Stacking +stack_dithers = [ + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", "EXPTIME", TARGET_KEY, "SUBCOORD"]), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=True, + subtract_bkg=False, + cache=False, + center_type="ALL", + temp_output_sub_dir="stacks_weights", + header_keys_to_combine=["RAWID"], + ), +] + +first_pass_stacking = ( + astrometry_net + + astrometry_scamp + + first_pass_validate_astrometry_export_and_filter + + [ + ImageSaver(output_dir_name="fp_post_astrometry"), + ] + + stack_dithers + + [ + ImageSaver(output_dir_name="fp_stack"), + ] +) + +load_astrometried = [ + ImageLoader( + input_sub_dir="fp_post_astrometry", + input_img_dir=base_output_dir, + ) +] + +# Second pass calibration +second_pass_calibration = [ + ImageDebatcher(), + ImageBatcher([TARGET_KEY, "BOARD_ID"]), + Sextractor( + output_sub_dir="sp_stack_source_mask", + **sextractor_astrometry_config, + checkimage_type="SEGMENTATION", + cache=True, + ), + MaskPixelsFromPathInverted( + mask_path_key=sextractor_checkimg_map["SEGMENTATION"], + write_masked_pixels_to_file=True, + output_dir="sp_stack_source_mask", + ), + WriteMaskedCoordsToFile(output_dir="sp_stack_mask"), + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", TARGET_KEY, "SUBCOORD"]), + ReloadSwarpComponentImages( + copy_header_keys=FITS_MASK_KEY, + ), + LoadImageFromHeader( + header_key=RAW_IMG_KEY, + copy_header_keys=[SCAMP_HEADER_KEY, FITS_MASK_KEY], + ), + AstrometryFromFile(astrometry_file_key=SCAMP_HEADER_KEY), + ImageSaver(output_dir_name="sp_astrometry", write_mask=True), + MaskPixelsFromWCS( + write_masked_pixels_to_file=True, + output_dir="sp_source_mask", + only_write_mask=True, + ), + ImageSaver(output_dir_name="sp_masked", write_mask=True), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY), + ImageSaver(output_dir_name="sp_calibration_flat"), + Sextractor( + **sextractor_astrometry_config, + write_regions_bool=True, + output_sub_dir="skysub", + checkimage_type=["-BACKGROUND"], + ), + SextractorBkgSubtractor(), + ImageSaver(output_dir_name="skysub"), +] + +# Second pass astrometry +second_pass_astrometry = ( + astrometry_net + + [ + ImageSaver(output_dir_name="post_anet"), + ] + + astrometry_scamp + + [ + ImageSaver(output_dir_name="post_scamp"), + ] +) + +# Second pass stacking +second_pass_stack = ( + second_pass_astrometry + + second_pass_validate_astrometry_export_and_filter + + stack_dithers + + [ + ImageSaver(output_dir_name="stack"), + ] +) + +# Photometric calibration +photcal_and_export = [ + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", TARGET_KEY, "SUBCOORD"]), + HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), + CustomImageBatchModifier(masked_images_rejector), + Sextractor( + **sextractor_photometry_config, + output_sub_dir="stack_psf", + checkimage_type="BACKGROUND_RMS", + ), + PSFex(config_path=psfex_path, output_sub_dir="phot", norm_fits=True), + Sextractor( + **sextractor_photometry_psf_config, + output_sub_dir="phot", + checkimage_type="BACKGROUND_RMS", + use_psfex=True, + ), + CustomImageBatchModifier(winter_photometric_ref_catalog_namer), + PhotCalibrator( + ref_catalog_generator=winter_photometric_catalog_generator, + catalogs_purifier=winter_photometric_catalogs_purifier, + temp_output_sub_dir="phot", + write_regions=True, + cache=True, + zp_calculator=ZPWithColorTermCalculator( + color_colnames_guess_generator=winter_photcal_color_columns_generator, + reject_outliers=True, + solver="curve_fit", + ), + zp_column_name="MAG_AUTO", + ), + CatalogLimitingMagnitudeCalculator( + sextractor_mag_key_name="MAG_AUTO", write_regions=True + ), + AstrometryStatsWriter( + ref_catalog_generator=winter_astrometric_ref_catalog_generator, + image_catalog_purifier=winter_astrostat_catalog_purifier, + write_regions=True, + cache=False, + crossmatch_radius_arcsec=5.0, + ), + ImageSaver(output_dir_name="final"), + DatabaseImageInserter(db_table=Stack, duplicate_protocol="replace"), + ImageDatabaseMultiEntryUpdater( + sequence_key="rawid", + db_table=Raw, + db_alter_columns="ustackid", + ), + ImagePlotter( + output_sub_dir="final_stacks_plots", + annotate_fields=[ + BASE_NAME_KEY, + "COADDS", + TARGET_KEY, + "CRVAL1", + "CRVAL2", + "FILTER", + "ZP", + "ZPSTD", + ], + ), +] + +photcal_stacks = [ + ImageLoader( + input_sub_dir="stack", + input_img_dir=base_output_dir, + load_image=load_winter_stack, + ), +] + photcal_and_export + +# Final stack +load_final_stack = [ + ImageLoader( + input_sub_dir="final", + input_img_dir=base_output_dir, + load_image=load_winter_stack, + ), + DatabaseImageInserter(db_table=Stack, duplicate_protocol="ignore"), +] + +plot_stack = [ + ImageDebatcher(), + ImageBatcher([TARGET_KEY, "BOARD_ID"]), + ImagePlotter( + output_sub_dir="final_stacks_plots", + annotate_fields=[ + BASE_NAME_KEY, + "COADDS", + TARGET_KEY, + "CRVAL1", + "CRVAL2", + "FILTER", + "ZP", + "ZPSTD", + ], + ), +] + +# To make a mosaic by stacking all boards +stack_boards = [ + ImageBatcher([TARGET_KEY]), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=False, + subtract_bkg=False, + cache=False, + center_type="ALL", + temp_output_sub_dir="mosaic_weights", + ), + ImageSaver(output_dir_name="mosaic"), +] + +mosaic = load_final_stack + stack_boards + +# To make cals for focusing +focus_subcoord = [ + HeaderAnnotator(input_keys=["BOARD_ID"], output_key="SUBCOORD"), + HeaderAnnotator(input_keys=["BOARD_ID"], output_key="SUBDETID"), +] + +# End of image-reduction blocks diff --git a/mirar/pipelines/winter/blocks_subtraction.py b/mirar/pipelines/winter/blocks_subtraction.py new file mode 100644 index 000000000..a1e82ac53 --- /dev/null +++ b/mirar/pipelines/winter/blocks_subtraction.py @@ -0,0 +1,239 @@ +""" +Module with blocks for WINTER image subtraction and candidate processing +""" + +from mirar.catalog.kowalski import PS1, TMASS +from mirar.paths import SOURCE_HISTORY_KEY, SOURCE_NAME_KEY, TARGET_KEY, ZP_KEY +from mirar.pipelines.winter.config import ( + prv_candidate_cols, + psfex_path, + sextractor_candidate_config, + sextractor_reference_psf_phot_config, + swarp_config_path, + winter_avro_schema_path, + winter_fritz_config, +) +from mirar.pipelines.winter.generator import ( + mask_stamps_around_bright_stars, + winter_candidate_annotator_filterer, + winter_candidate_avro_fields_calculator, + winter_candidate_quality_filterer, + winter_history_deprecated_constraint, + winter_imsub_catalog_purifier, + winter_new_source_updater, + winter_reference_generator, + winter_reference_image_resampler_for_zogy, + winter_reference_psf_phot_sextractor, + winter_reference_psfex, + winter_reference_sextractor, + winter_skyportal_annotator, + winter_source_entry_updater, +) +from mirar.pipelines.winter.models import ( + NAME_START, + SOURCE_PREFIX, + Candidate, + Diff, + Source, +) +from mirar.processors.astromatic import PSFex +from mirar.processors.astromatic.sextractor.sextractor import Sextractor +from mirar.processors.avro import IPACAvroExporter +from mirar.processors.database.database_inserter import ( + DatabaseImageInserter, + DatabaseSourceInserter, +) +from mirar.processors.database.database_selector import ( + SelectSourcesWithMetadata, + SingleSpatialCrossmatchSource, +) +from mirar.processors.mask import MaskPixelsFromFunction +from mirar.processors.photometry import AperturePhotometry, PSFPhotometry +from mirar.processors.reference import GetReferenceImage, ProcessReference +from mirar.processors.skyportal.skyportal_candidate import SkyportalCandidateUploader +from mirar.processors.sources import ( + CandidateNamer, + CustomSourceTableModifier, + SourceLoader, + SourceWriter, + ZOGYSourceDetector, +) +from mirar.processors.split import SUB_ID_KEY, SwarpImageSplitter +from mirar.processors.utils import ( + HeaderAnnotator, + ImageBatcher, + ImageDebatcher, + ImageLoader, + ImageSaver, +) +from mirar.processors.xmatch import XMatch +from mirar.processors.zogy.reference_aligner import AlignReference +from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare + +# Reference building +refbuild = [ + GetReferenceImage(ref_image_generator=winter_reference_generator), + ImageSaver(output_dir_name="stacked_ref"), +] + +# Image subtraction +split_stack = [ + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", TARGET_KEY, "SUBCOORD", "STACKID"]), + SwarpImageSplitter(swarp_config_path=swarp_config_path, n_x=2, n_y=1), + ImageSaver(output_dir_name="split_stacks"), +] + +imsub = [ + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", TARGET_KEY, "SUBCOORD", "STACKID"]), + HeaderAnnotator(input_keys=[SUB_ID_KEY], output_key="SUBDETID"), + ProcessReference( + ref_image_generator=winter_reference_generator, + swarp_resampler=winter_reference_image_resampler_for_zogy, + sextractor=winter_reference_sextractor, + ref_psfex=winter_reference_psfex, + phot_sextractor=winter_reference_psf_phot_sextractor, + ), + Sextractor( + **sextractor_reference_psf_phot_config, + output_sub_dir="subtract", + cache=False, + use_psfex=True, + ), + PSFex(config_path=psfex_path, output_sub_dir="subtract", norm_fits=True), + AlignReference( + order=1, + sextractor=winter_reference_sextractor, + psfex=winter_reference_psfex, + phot_sextractor=winter_reference_psf_phot_sextractor, + catalog_purifier=winter_imsub_catalog_purifier, + ), + MaskPixelsFromFunction(mask_function=mask_stamps_around_bright_stars), + ImageSaver(output_dir_name="presubtract"), + ZOGYPrepare( + output_sub_dir="subtract", + sci_zp_header_key="ZP_AUTO", + ref_zp_header_key=ZP_KEY, + catalog_purifier=winter_imsub_catalog_purifier, + x_key="XMODEL_IMAGE", + y_key="YMODEL_IMAGE", + flux_key="FLUX_POINTSOURCE", + ), + # ImageSaver(output_dir_name="prezogy"), + ZOGY( + output_sub_dir="subtract", sci_zp_header_key="ZP_AUTO", ref_zp_header_key=ZP_KEY + ), + ImageSaver(output_dir_name="diffs"), + DatabaseImageInserter(db_table=Diff, duplicate_protocol="replace"), + ImageSaver(output_dir_name="subtract"), +] + +load_sub = [ + ImageLoader(input_sub_dir="subtract"), +] +detect_candidates = [ + ZOGYSourceDetector( + output_sub_dir="subtract", + **sextractor_candidate_config, + write_regions=True, + detect_negative_sources=True, + ), + PSFPhotometry(phot_cutout_half_size=10), + AperturePhotometry( + temp_output_sub_dir="aper_photometry", + aper_diameters=[8, 16], + phot_cutout_half_size=50, + bkg_in_diameters=[25, 25], + bkg_out_diameters=[40, 40], + col_suffix_list=["", "big"], + ), + CustomSourceTableModifier(winter_candidate_annotator_filterer), + SourceWriter(output_dir_name="candidates"), +] + +load_sources = [ + SourceLoader(input_dir_name="candidates"), +] + +crossmatch_candidates = [ + XMatch(catalog=TMASS(num_sources=3, search_radius_arcmin=0.5)), + XMatch(catalog=PS1(num_sources=3, search_radius_arcmin=0.5)), + SourceWriter(output_dir_name="kowalski"), + CustomSourceTableModifier( + modifier_function=winter_candidate_avro_fields_calculator + ), +] + +select_history = [ + SelectSourcesWithMetadata( + db_query_columns=["sourceid"], + db_table=Candidate, + db_output_columns=prv_candidate_cols + [SOURCE_NAME_KEY], + base_output_column=SOURCE_HISTORY_KEY, + additional_query_constraints=winter_history_deprecated_constraint, + ), +] + +name_candidates = ( + [ + # Check if the source is already in the source table + SingleSpatialCrossmatchSource( + db_table=Source, + db_output_columns=["sourceid", SOURCE_NAME_KEY], + crossmatch_radius_arcsec=2.0, + ra_field_name="average_ra", + dec_field_name="average_dec", + ), + # Assign names to the new sources + CandidateNamer( + db_table=Source, + base_name=SOURCE_PREFIX, + name_start=NAME_START, + db_name_field=SOURCE_NAME_KEY, + ), + # Add the new sources to the source table + CustomSourceTableModifier(modifier_function=winter_new_source_updater), + DatabaseSourceInserter( + db_table=Source, + duplicate_protocol="ignore", + ), + # Get all candidates associated with source + ] + + select_history + + [ + # Update average ra and dec for source + CustomSourceTableModifier(modifier_function=winter_source_entry_updater), + # Update sources in the source table + DatabaseSourceInserter( + db_table=Source, + duplicate_protocol="replace", + ), + # Add candidates in the candidate table + DatabaseSourceInserter( + db_table=Candidate, + duplicate_protocol="fail", + ), + SourceWriter(output_dir_name="preavro"), + ] +) + +avro_export = [ + IPACAvroExporter( + topic_prefix="winter", + base_name="WNTR", + broadcast=False, + avro_schema_path=winter_avro_schema_path, + ), + CustomSourceTableModifier(modifier_function=winter_candidate_quality_filterer), + SourceWriter(output_dir_name="preskyportal"), +] + +process_candidates = crossmatch_candidates + name_candidates + avro_export + +load_skyportal = [SourceLoader(input_dir_name="preskyportal")] + +send_to_skyportal = [ + CustomSourceTableModifier(modifier_function=winter_skyportal_annotator), + SkyportalCandidateUploader(**winter_fritz_config), +] diff --git a/mirar/pipelines/winter/models/__init__.py b/mirar/pipelines/winter/models/__init__.py index 3f20467ae..305718516 100644 --- a/mirar/pipelines/winter/models/__init__.py +++ b/mirar/pipelines/winter/models/__init__.py @@ -26,6 +26,10 @@ FiltersTable, populate_filters, ) +from mirar.pipelines.winter.models._fp_astrometry_stats import ( + FirstPassAstrometryStat, + FirstPassAstrometryStatsTable, +) from mirar.pipelines.winter.models._img_type import ( ALL_ITID, ImgType, diff --git a/mirar/pipelines/winter/models/_fp_astrometry_stats.py b/mirar/pipelines/winter/models/_fp_astrometry_stats.py new file mode 100644 index 000000000..63d94af21 --- /dev/null +++ b/mirar/pipelines/winter/models/_fp_astrometry_stats.py @@ -0,0 +1,94 @@ +""" +Models for the 'fp_astrometry_stats' table +""" + +import logging +from typing import ClassVar + +from pydantic import Field +from sqlalchemy import VARCHAR, Column, Float, ForeignKey +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mirar.database.base_model import BaseDB, dec_field, ra_field +from mirar.pipelines.winter.models._raw import RawsTable +from mirar.pipelines.winter.models.base_model import WinterBase + +logger = logging.getLogger(__name__) + + +class FirstPassAstrometryStatsTable( + WinterBase +): # pylint: disable=too-few-public-methods + """ + Astrometry stats after first pass processing table in database + """ + + __tablename__ = "fp_astrometry_stats" + __table_args__ = {"extend_existing": True} + + rawid = mapped_column(ForeignKey("raws.rawid"), primary_key=True, unique=True) + fp_astrom_raw_ids: Mapped["RawsTable"] = relationship( + back_populates="fp_astrometry" + ) + savepath = Column(VARCHAR(255), unique=True) + crval1 = Column(Float) + crval2 = Column(Float) + crpix1 = Column(Float) + crpix2 = Column(Float) + cd1_1 = Column(Float) + cd1_2 = Column(Float) + cd2_1 = Column(Float) + cd2_2 = Column(Float) + astunc = Column(Float) + astunc95 = Column(Float) + astfield = Column(Float) + fwhm_med = Column(Float) + fwhm_std = Column(Float) + fwhm_pix = Column(Float) + astirms1 = Column(Float) + astirms2 = Column(Float) + astrrms1 = Column(Float) + astrrms2 = Column(Float) + + ra_column_name = "crval1" + dec_column_name = "crval2" + + +default_unknown_field = Field(default=-999) + + +class FirstPassAstrometryStat(BaseDB): + """ + A pydantic model for a first-pass astrometry stats database entry + """ + + sql_model: ClassVar = FirstPassAstrometryStatsTable + + rawid: int = Field(ge=0) + savepath: str = Column(VARCHAR(255), unique=True) + crval1: float = ra_field + crval2: float = dec_field + crpix1: float = default_unknown_field + crpix2: float = default_unknown_field + cd1_1: float = default_unknown_field + cd1_2: float = default_unknown_field + cd2_1: float = default_unknown_field + cd2_2: float = default_unknown_field + astunc: float = default_unknown_field + astunc95: float = default_unknown_field + astfield: float = default_unknown_field + fwhm_med: float = default_unknown_field + fwhm_std: float = default_unknown_field + fwhm_pix: float = default_unknown_field + astirms1: float = default_unknown_field + astirms2: float = default_unknown_field + astrrms1: float = default_unknown_field + astrrms2: float = default_unknown_field + + def exists(self) -> bool: + """ + Checks if the pydantic-ified data exists the corresponding sql database + + :return: bool + """ + return self._exists(values=self.rawid, keys="rawid") diff --git a/mirar/pipelines/winter/models/_raw.py b/mirar/pipelines/winter/models/_raw.py index 98d6d58bb..6a07362d2 100644 --- a/mirar/pipelines/winter/models/_raw.py +++ b/mirar/pipelines/winter/models/_raw.py @@ -46,6 +46,10 @@ class RawsTable(WinterBase): # pylint: disable=too-few-public-methods back_populates="astrom_raw_ids" ) + fp_astrometry: Mapped["FirstPassAstrometryStatsTable"] = relationship( + back_populates="fp_astrom_raw_ids" + ) + class Raw(BaseDB): """ diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index ea4db9f31..9e89d3a41 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -9,27 +9,20 @@ from mirar.io import open_mef_image from mirar.pipelines.base_pipeline import Pipeline from mirar.pipelines.winter.blocks import ( - astrometry, build_test, csvlog, detect_candidates, detrend_unpacked, diff_forced_photometry, extract_all, + first_pass_processing, focus_cals, full_reduction, imsub, load_calibrated, - load_final_stack, load_raw, - load_skyportal, load_test, mask_and_split, - mosaic, - name_candidates, - only_ref, - photcal_stacks, - plot_stack, process_candidates, realtime, reduce, @@ -38,15 +31,27 @@ refbuild, reftest, save_raw, - select_history, - select_split_subset, - send_to_skyportal, + second_pass_astrometry, + second_pass_processing, stack_forced_photometry, unpack_all, unpack_all_no_calhunter, unpack_subset, unpack_subset_no_calhunter, ) +from mirar.pipelines.winter.blocks_reduction import ( + load_final_stack, + mosaic, + photcal_stacks, + plot_stack, + select_split_subset, +) +from mirar.pipelines.winter.blocks_subtraction import ( + load_skyportal, + name_candidates, + select_history, + send_to_skyportal, +) from mirar.pipelines.winter.config import PIPELINE_NAME, winter_cal_requirements from mirar.pipelines.winter.load_winter_image import load_raw_winter_mef @@ -62,7 +67,7 @@ class WINTERPipeline(Pipeline): default_cal_requirements = winter_cal_requirements all_pipeline_configurations = { - "astrometry": load_calibrated + astrometry, + "astrometry": load_calibrated + second_pass_astrometry, "unpack_subset": unpack_subset, "unpack_all": unpack_all, "unpack_subset_no_calhunter": unpack_subset_no_calhunter, @@ -87,7 +92,6 @@ class WINTERPipeline(Pipeline): + process_candidates, "refbuild": refbuild, "reftest": reftest, - "only_ref": only_ref, "realtime": realtime, "detect_candidates": load_final_stack + imsub + detect_candidates, "full_imsub": load_final_stack + imsub + detect_candidates + process_candidates, @@ -108,6 +112,8 @@ class WINTERPipeline(Pipeline): "name_candidates": name_candidates, "diff_forced_phot": diff_forced_photometry, "stack_forced_phot": stack_forced_photometry, + "firstpass": first_pass_processing, + "secpass": second_pass_processing, "detrend": unpack_all + detrend_unpacked, "send_with_history": select_history + send_to_skyportal, } diff --git a/mirar/processors/astromatic/swarp/component_images.py b/mirar/processors/astromatic/swarp/component_images.py index 8e39d13ce..e7a132c7a 100644 --- a/mirar/processors/astromatic/swarp/component_images.py +++ b/mirar/processors/astromatic/swarp/component_images.py @@ -7,14 +7,12 @@ from pathlib import Path import numpy as np -from astropy.io.fits import Header -from mirar.data import ImageBatch -from mirar.io import open_fits +from mirar.data import Image, ImageBatch +from mirar.io import open_raw_image from mirar.paths import STACKED_COMPONENT_IMAGES_KEY from mirar.processors.astromatic.swarp.swarp import Swarp from mirar.processors.base_processor import BaseImageProcessor, PrerequisiteError -from mirar.processors.utils.image_saver import ImageSaver logger = logging.getLogger(__name__) @@ -28,7 +26,7 @@ class ReloadSwarpComponentImages(BaseImageProcessor): def __init__( self, - load_image: Callable[[str], [np.ndarray, Header]] = open_fits, + load_image: Callable[[str], [Image]] = open_raw_image, header_key=STACKED_COMPONENT_IMAGES_KEY, copy_header_keys: str | list[str] = None, ): @@ -81,16 +79,3 @@ def check_prerequisites( ) logger.error(err) raise PrerequisiteError(err) - - index = np.argmax(mask) - - preceding_step = self.preceding_steps[index - 1] - - if not isinstance(preceding_step, ImageSaver): - err = ( - f"{self.__module__} requires an {ImageSaver} to be used to save the " - f"component images immediately before {Swarp} is run. " - f"However, the following steps were found: {self.preceding_steps}." - ) - logger.error(err) - raise PrerequisiteError(err) diff --git a/tests/test_winter_pipeline.py b/tests/test_winter_pipeline.py index 7289da82d..d9a526ccc 100644 --- a/tests/test_winter_pipeline.py +++ b/tests/test_winter_pipeline.py @@ -13,458 +13,379 @@ logger = logging.getLogger(__name__) expected_zp = { - "ZP_2.0": 23.959992146021143, - "ZP_2.0_std": 0.04468593502379257, - "ZP_2.0_nstars": 118, - "ZP_3.0": 24.488647137271013, - "ZP_3.0_std": 0.043678000162985946, - "ZP_3.0_nstars": 118, - "ZP_4.0": 24.67403631422544, - "ZP_4.0_std": 0.040829710238539714, - "ZP_4.0_nstars": 118, - "ZP_5.0": 24.736553404147642, - "ZP_5.0_std": 0.040354725833759444, - "ZP_5.0_nstars": 118, - "ZP_6.0": 24.77188170392595, - "ZP_6.0_std": 0.039765732833753016, - "ZP_6.0_nstars": 118, - "ZP_7.0": 24.782397765144424, - "ZP_7.0_std": 0.04069653155959119, - "ZP_7.0_nstars": 118, - "ZP_8.0": 24.799413106226112, - "ZP_8.0_std": 0.040876147446689284, - "ZP_8.0_nstars": 118, - "ZP_AUTO": 24.787539557156656, - "ZP_AUTO_std": 0.04316430194985392, - "ZP_AUTO_nstars": 118, - "ZP_PSF": 24.680094141593663, - "ZP_PSF_std": 0.03875121055985624, - "ZP_PSF_nstars": 117, - "SCORMEAN": -0.07678255815161257, - "SCORMED": -0.07307016879473982, - "SCORSTD": 1.2836771224604269, + "ZP_2.0": 24.05931984677565, + "ZP_2.0_std": 0.03830265576481768, + "ZP_2.0_nstars": 114, + "ZP_3.0": 24.5182462444065, + "ZP_3.0_std": 0.03897102606054595, + "ZP_3.0_nstars": 113, + "ZP_4.0": 24.66877193729583, + "ZP_4.0_std": 0.04413923966098385, + "ZP_4.0_nstars": 115, + "ZP_5.0": 24.735337207194643, + "ZP_5.0_std": 0.04470635796321621, + "ZP_5.0_nstars": 115, + "ZP_6.0": 24.756819402061478, + "ZP_6.0_std": 0.045164962968125014, + "ZP_6.0_nstars": 115, + "ZP_7.0": 24.77976130514258, + "ZP_7.0_std": 0.04516449922952734, + "ZP_7.0_nstars": 115, + "ZP_8.0": 24.791871235399995, + "ZP_8.0_std": 0.045112750520510565, + "ZP_8.0_nstars": 115, + "ZP_AUTO": 24.79282472222403, + "ZP_AUTO_std": 0.045853218043986683, + "ZP_AUTO_nstars": 115, + "ZP_PSF": 24.728054363490656, + "ZP_PSF_std": 0.037042100713524956, + "ZP_PSF_nstars": 111, + "SCORMEAN": -0.08252085765164723, + "SCORMED": -0.06930644943369992, + "SCORSTD": 1.2697482555083541, } expected_dataframe_values = { "magpsf": [ - 17.27350131218496, - 17.09322645828431, - 15.713888241482717, - 17.105224962013885, - 17.008938740753777, - 15.384459619262936, - 17.21898287647103, - 16.04271800524978, - 16.668565855452023, - 16.810454279339858, - 17.307242451093423, - 16.972841300543138, - 16.941287839498322, - 15.223083862034935, - 13.751437577040493, - 16.721470324891584, - 14.638107336381436, - 17.210166300948103, - 17.307626137920188, - 16.893941199130477, - 16.97220970929738, - 17.118547153712985, - 16.8317526638244, - 16.89344613545377, - 17.156920944909402, - 15.542696944225593, - 17.165647189898216, - 16.79142879764631, - 16.99407602584789, - 15.035329840583662, - 17.298008077284045, - 16.274824907777507, - 17.161498974492712, - 17.013033586359, - 13.96106857700322, - 16.906071568137946, - 17.229220094494575, - 17.324247707038786, - 17.136314682008248, - 17.13753763323972, - 16.612709667108163, - 16.807132967300284, - 16.778558453207374, - 16.978011370942525, - 13.100279682962327, - 16.760287067600288, - 17.008749910774725, - 17.21886566185933, - 16.58143095026838, - 17.339485605257934, - 14.486265267643333, - 17.243868923747275, - 13.965231872671005, - 17.26807284961, - 16.97636957860883, - 17.081314430162887, - 17.105877710132656, - 16.84434160574626, - 17.189958568056056, - 16.805993874359093, - 16.8805695032575, - 17.211610242549998, - 16.43629525133231, - 17.155483309634597, - 15.13199772235674, - 16.62824806836388, - 14.857299986990052, - 17.04241903537911, - 16.744116033343737, - 16.41624822749315, - 14.26243887701997, - 17.074136553349668, - 16.672317215993008, - 16.893263805327962, - 16.42420572240269, - 16.757310821385687, - 17.103769273690183, - 16.89459390466265, - 17.01645956685531, - 16.719422232121413, - 17.151449988669178, - 16.900300833319736, - 16.69076883805588, - 17.28809534037981, - 16.754413508000965, - 17.104308793437497, - 16.612828734865637, - 16.900141299457623, - 16.6391312226793, - 17.107340657213395, - 17.089047823244083, - 16.724590419651946, - 16.46393866733439, - 17.321202163947223, - 15.813487767836511, - 16.797094807489362, - 16.963478321842267, - 17.064810341495168, - 16.757008929926098, - 16.65536720973002, - 16.869352917812343, - 16.97436057632337, - 16.622057226326298, - 16.891273171408326, - 17.07066160720414, - 16.544030066057203, - 17.032793094264715, - 16.912994034129305, - 17.141840188532274, - 16.99011899510101, - 17.120004207653388, - 16.347099971231728, - 16.992222909352044, - 16.364518012311123, - 16.610166854604763, - 14.708637520933033, - 16.526627668909583, - 17.32013356844476, - 17.180532478594348, - 15.547793156348362, - 17.144041498725002, - 17.241753581308775, - 16.956899051161265, - 17.057464889037107, - 16.429022270236015, - 16.991958586265362, - 17.039656662375105, - 15.390701791995248, - 16.454094068423558, - 16.71984887778577, - 17.105512796207712, - 14.057845718524817, - 17.017179945965065, - 15.60804493249878, - 14.277153007166667, - 17.03857787596209, - 17.37886878151883, - 17.19546353377233, - 16.795133109068068, - 17.12263253458518, - 16.597136534501917, - 17.01202855519201, - 17.207657288209354, - 16.491983482646773, - 17.09281574634499, - 16.571826955219493, - 17.22561419079348, - 16.990088350047365, - 16.95122402738543, - 17.165326954959383, - 17.206268592027268, - 16.163269971087185, - 16.660481923377084, - 16.712269366328478, - 16.411126093164874, - 14.860312000315602, - 17.29503147343265, - 16.47807086702509, - 16.40641429557884, - 17.383199141593817, - 17.337987820202375, - 17.032551295224344, - 16.475874862095967, - 15.793724489756446, - 16.66017350150458, - 16.51142686447066, - 17.161287897560527, - 16.38003477240683, - 15.296467778251994, - 17.307921767206196, - 16.56741414443544, - 16.636337718528587, - 17.231137978873, - 17.19155923777967, - 16.792077535991666, - 16.87787769106422, - 17.372787091325524, - 17.288561211533498, - 17.105693799248478, - 16.668231155666128, - 17.169081638125856, - 14.159432439885103, - 17.361250773358222, - 17.255655722557798, - 17.25993916765181, - 17.363251253957493, - 17.2730760717391, - 16.96062660802835, - 17.08512904568409, - 17.3102682168863, - 16.024701315353695, - 17.255079863682376, - 17.25436960173019, - 17.34550256637912, - 16.698421202393032, - 17.35051421743254, - 17.274696412559134, - 17.193269986322242, - 17.14131601211877, - 17.240933903357508, - 17.36379545931206, - 17.001411691837372, - 17.225180821684507, - 16.35910321787454, - 17.34600973183101, - 17.18098893490146, - 16.837761545359154, + 17.080945751150622, + 14.708106222027595, + 17.022750119829055, + 16.610904782427035, + 17.00821547529815, + 15.083344543039614, + 17.20262789934139, + 16.577858978827802, + 17.158604324326824, + 17.199527444628906, + 17.27594948160655, + 16.439715610781313, + 16.74298394575055, + 16.90284959273128, + 15.361447411583198, + 13.403504301394129, + 15.900416455898991, + 16.671780747005037, + 16.729265975344596, + 17.063889137217537, + 17.330950249758722, + 15.448761709210673, + 16.97604942403651, + 15.974175632636472, + 16.936752852987013, + 17.10199694292421, + 16.94319319035184, + 17.36789780216876, + 16.34422475877286, + 17.032302513390142, + 13.799390633284723, + 16.891713940452558, + 17.38269290373386, + 17.210960145015548, + 17.13774520911883, + 16.89176826811356, + 17.017207170910268, + 16.88725093350201, + 13.03394604219924, + 13.441959283137027, + 16.804435662879868, + 17.172036149770882, + 16.709798485433048, + 17.004112384222225, + 17.3089991836995, + 16.896276973809492, + 17.062648930268136, + 14.165765950329677, + 16.816833387421166, + 17.25798634387369, + 16.472393342354607, + 16.12811580044614, + 16.913044144352817, + 16.268274912394237, + 17.080148820424736, + 13.47613884658073, + 16.425583658844143, + 16.75108946818958, + 16.755069641373844, + 16.717891550554384, + 16.72425486563276, + 16.462018948904134, + 16.926717165375624, + 16.610232516100197, + 16.357849492106514, + 16.6979110423582, + 16.518174380369885, + 16.74647586576681, + 17.007835404136856, + 16.844111593569146, + 16.8340556097207, + 16.732168332845042, + 17.08749231101348, + 17.004516094731088, + 16.939554063633867, + 17.107446123685204, + 16.809913610091975, + 17.162522468430105, + 17.121368200546932, + 16.678845076290756, + 16.83265560516275, + 16.40310284029951, + 16.339574015662656, + 16.103111705333873, + 17.15340493534194, + 16.896123215615198, + 16.40243616039653, + 17.1582931546209, + 16.459604745207848, + 17.19450107263499, + 17.080459984705314, + 15.385225939178683, + 16.752597031290563, + 16.054354498228832, + 16.892133609291236, + 17.21063020807636, + 16.843345213298324, + 17.046073451061847, + 15.083932309155868, + 17.005670537635034, + 16.502765523687053, + 17.233621983138807, + 16.25253991989338, + 16.792475363556207, + 17.166090455912737, + 17.184076767955517, + 14.161989445395921, + 13.915740806363097, + 17.015245779545417, + 17.14100241170922, + 17.02748021543208, + 16.473754038876688, + 16.95371446975483, + 17.25862764580871, + 16.970047144963175, + 16.456309223219534, + 16.795726077554612, + 16.724194652456312, + 17.03003655146124, + 17.371117319852733, + 13.207472235771684, + 16.825596397232204, + 16.921597421752924, + 16.627804357173176, + 17.173908929043368, + 17.289434436432987, + 16.860161926238483, + 16.67987198304645, + 16.350044321461816, + 16.64659657729659, + 16.334020989295787, + 16.955573136373012, + 17.00390944462132, + 16.828263678065163, + 16.730433842348425, + 17.0782404690106, + 17.084119394019353, + 16.73527213591415, + 17.12199043952996, + 17.04345172487818, + 17.3442701053555, + 17.001986744948834, + 16.824830565114077, + 16.94922343991597, + 17.2948269136125, + 17.093191747370042, + 17.273424922851895, + 17.091147256988588, + 16.996458095823126, + 17.35204129179263, + 17.050534676211765, + 14.613900678468525, + 16.980103906679226, + 17.289130974003953, + 17.060689510213372, + 17.105322606609835, + 16.9180294198705, + 14.25804335077192, + 16.925405738907504, + 16.63195525193173, + 17.317228264482818, + 11.977605148619725, + 17.124705534009276, + 14.363627424786785, + 16.61936326494321, + 17.34971534768361, + 17.221842755687632, + 16.797271781138946, ], "magap": [ - 16.999210731347183, - 17.100506738979092, - 16.25159341168322, - 16.704145608322463, - 16.623647691100352, - 15.98980930450428, - 16.87015054985482, - 16.02597923345043, - 16.275233646951982, - 16.74383538596543, - 17.647757293256454, - 17.09282347300866, - 16.744496325658893, - 15.518405450926448, - 14.309880578392391, - 16.26954128521023, - 14.952411884625167, - 16.966855980892113, - 16.88418495250187, - 16.65862012697983, - 16.39227583759153, - 16.72395009834748, - 16.70002860077116, - 16.591802143051545, - 16.72826703519212, - 15.75867416567782, - 17.005281640018126, - 17.22746035015709, - 16.678239127573978, - 15.836790848315239, - 17.12090037510687, - 16.465734773962037, - 16.935769537336146, - 16.259178856872225, - 14.484965766820771, - 16.760572075770355, - 16.982127026000374, - 17.740498463341915, - 16.605728759755106, - 17.147616902084096, - 17.175760452331538, - 16.54181036912889, - 15.912919124181482, - 17.380202483331704, - 12.460595564712591, - 16.48649676072447, - 16.742543966226886, - 17.20630971328091, - 16.40313452206932, - 16.929882768590964, - 14.994085531851145, - 17.07864666765043, - 14.02502197092328, - 17.971362389118152, - 17.160275837363063, - 16.99415140475904, - 16.523178250853473, - 15.959588041542268, - 17.260628788146256, - 16.16745653703493, - 17.161032491379316, - 17.66528236019196, - 16.243587348509966, - 16.86861084953622, - 15.580812147424217, - 16.053404932964376, - 15.34452757018662, - 16.148079216837182, - 17.18431781226372, - 16.320687998422603, - 14.266836765828932, - 16.758444646216955, - 16.77467801195104, - 17.05608223434902, - 15.73395530811009, - 17.015459439925706, - 17.29014266066559, - 16.31469165065115, - 16.799784945061383, - 16.774291503329813, - 16.37974871580147, - 16.31531176049583, - 17.1030415887711, - 17.16785642006961, - 16.318416329812617, - 16.851157903369177, - 15.867077544746225, - 17.519553472751983, - 15.894054604738814, - 16.659287123346694, - 16.68566096707504, - 16.214984541409052, - 16.53832577456164, - 16.71266722711639, - 16.541180580849627, - 16.052911061331667, - 16.65635938418327, - 16.536996244443742, - 16.93993244166624, - 16.176178937268993, - 17.538415418061113, - 16.232060365924923, - 15.902185515336996, - 16.324738148695005, - 16.380296763947307, - 16.04756348230723, - 16.572541913219872, - 16.325487478985714, - 17.011510218099474, - 16.572837583899048, - 16.64579057973391, - 15.922008941550265, - 16.388625323762714, - 15.851447557808802, - 16.830973717564035, - 15.419639475303454, - 16.72251375531153, - 16.62748273774924, - 17.078327379173224, - 16.27655154551509, - 16.70933773621512, - 16.64293859531479, - 16.173140640747565, - 16.961359265955828, - 16.360084218261, - 16.87991959142907, - 17.055124614969863, - 15.50104065405109, - 16.417805908956474, - 16.133873475515863, - 16.814569636007427, - 14.660498536518183, - 16.68800963918249, - 16.55849295650539, - 14.963221579014888, - 16.276458637004538, - 17.40730620250493, - 16.60752643771458, - 16.811186219187135, - 16.9684906789078, - 16.636508843186654, - 16.718545171763967, - 16.60223979607211, - 16.054318908101873, - 16.989437707314657, - 15.596521215384879, - 17.050769808745972, - 17.319690338292, - 17.151261459304543, - 17.319593645365984, - 16.886833407451306, - 16.95215363482017, - 16.027636203153644, - 15.980220552026504, - 15.881684491203925, - 15.4743924213499, - 16.564531629235148, - 16.299262248524634, - 16.21685541608992, - 17.381478605460394, - 16.495106361874978, - 16.413861981051333, - 16.457837261445725, - 16.392955897668347, - 15.958172956999954, - 15.964388058558105, - 16.682171529008603, - 15.649098487634674, - 15.468182632024932, - 17.756744586061753, - 16.407591849927833, - 16.59901640276672, - 16.722100413076603, - 16.49235350681755, - 16.592921876788324, - 16.9284738181111, - 17.02564344673953, - 16.920589465939837, - 17.02808439026945, - 16.041164546651622, - 16.983571765174375, - 14.59474753733716, - 16.817341868704467, - 17.228777534324674, - 16.794731461993205, - 17.028336026747013, - 17.14393185333722, - 16.29607018205465, - 16.336907648946937, - 17.739993248742056, - 16.567650122625132, - 17.116521421572653, - 17.199100668947292, - 17.030353348948786, - 16.61400766974273, - 16.664344870750213, - 16.83790778817827, - 16.540652799157126, - 16.807553642466875, - 17.18683244338872, - 16.628219705657465, - 16.923964861822743, - 17.07671419930161, - 16.287319608443802, - 17.565652285258874, - 17.095585391781697, - 16.138942839551547, + 16.87462708006406, + 15.445316092168218, + 16.53502410000455, + 16.872505578594605, + 17.07791801758546, + 14.88310366512772, + 17.3153096998834, + 16.300301355617016, + 17.1890094881321, + 16.50051339289036, + 17.159360313708852, + 16.380488838505187, + 16.334410568585582, + 16.51438903772086, + 15.358232592252131, + 14.048450482049532, + 16.570897952031537, + 16.793064766811668, + 17.318113590735997, + 16.73386038486479, + 17.547275598485243, + 15.7757855441433, + 16.707730511514146, + 15.982020059352218, + 17.245711924928926, + 17.23652169796587, + 16.72344146888542, + 17.76821145174536, + 16.70603401528382, + 16.898418033208824, + 14.305085355801188, + 16.592692192574187, + 16.562477079918963, + 17.00370024133123, + 16.768273751380647, + 16.807450650187107, + 17.044837209854133, + 17.34454075853897, + 12.607325077998224, + 13.753421468091428, + 16.084975225549428, + 17.155416440067505, + 16.76966340041968, + 16.399964540717654, + 17.052680868051894, + 17.511190369031432, + 16.648898897170398, + 14.938491798689634, + 16.811322459458633, + 16.737275407920816, + 16.31536466625427, + 14.562537369131663, + 16.757921934687882, + 16.43328387734389, + 17.00860950270895, + 14.32907965593547, + 15.719589055184226, + 16.908650205915656, + 16.222960638872884, + 16.19975010564343, + 16.445242676622506, + 16.18357700711738, + 16.299595509403517, + 16.95815415066128, + 17.125802924283413, + 16.958943025601723, + 16.059200014423133, + 16.614094217101023, + 16.727398930456772, + 16.30942643739309, + 16.705871095825547, + 16.470904345526833, + 17.519529126546878, + 16.5113519271329, + 16.730837568185592, + 17.012498339203937, + 15.907758916526598, + 17.50880197598907, + 16.60335987759278, + 16.00533195012615, + 16.354004429455426, + 17.21217732663935, + 15.992988052475422, + 15.921949087103531, + 16.573889082751027, + 16.761625157824014, + 16.408863999233052, + 16.399596552246376, + 16.355299069478818, + 16.708587413266166, + 16.31024801018228, + 15.859916551889343, + 16.91703574897814, + 16.19254397136892, + 16.65842608103067, + 16.88602319911652, + 16.20988737341061, + 16.70726583782669, + 13.883652898311016, + 16.544499190350805, + 16.160956699505434, + 16.59905636305212, + 15.949890081823035, + 16.796046606215004, + 17.050682216800272, + 16.585734955892196, + 14.769396110002456, + 14.480123632813251, + 16.498875286185196, + 16.427396551582373, + 16.744876864104157, + 16.565014011158624, + 16.554048235999907, + 16.93145002284904, + 16.77255335482007, + 16.390504179564115, + 16.172484552502212, + 16.232768473465804, + 17.261886796275203, + 16.98391807188197, + 13.682144100490515, + 16.378972887509235, + 17.04749081371895, + 15.996486105294599, + 16.789513548871145, + 16.756020451725817, + 16.40139117021704, + 16.678457727006958, + 16.03690171084004, + 17.218450328647183, + 15.925774039920292, + 16.35826052058541, + 16.435210613410298, + 17.070500194584664, + 16.861410184500308, + 17.295098480954593, + 16.96170662032879, + 15.835611350271234, + 16.22430990615117, + 17.102714614440387, + 16.93607719052254, + 16.323178433521235, + 16.058902571258926, + 16.361095001064143, + 16.601789454000375, + 16.801470476229152, + 16.77792599346344, + 16.924407617510553, + 16.319508525640792, + 17.903056284268583, + 16.665464067147234, + 15.402522385858607, + 16.06738401012889, + 17.24110295649703, + 16.81910696957604, + 17.212357914985, + 16.68381039202975, + 14.958763463401759, + 16.549400052944115, + 15.915883440199979, + 17.04538890974478, + 12.026487156686375, + 17.460936402417772, + 14.972024545250912, + 16.2531987291213, + 16.74351684436303, + 17.435456010551267, + 16.76502361691559, ], } - pipeline = get_pipeline( instrument="winter", selected_configurations=["test"], night="20230726" ) @@ -538,7 +459,7 @@ def test_pipeline(self): candidates_table = source_table.get_data() - self.assertEqual(len(candidates_table), 207) + self.assertEqual(len(candidates_table), 168) for key, value in expected_dataframe_values.items(): if isinstance(value, list): for ind, val in enumerate(value):