From 84d8363269e240c22cb91ea7d2f132aca9c04516 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Wed, 12 Feb 2025 18:57:29 +0100 Subject: [PATCH] ran black --- scripts/python/canfar_avail_results.py | 26 ++- scripts/python/canfar_run_analyse.py | 40 +++- scripts/python/cfis_field_select.py | 45 +++-- scripts/python/create_sample_results.py | 22 ++- scripts/python/create_star_cat.py | 8 +- scripts/python/get_number_objects.py | 6 +- scripts/python/merge_final_cat.py | 6 +- scripts/python/stats_global.py | 22 ++- scripts/python/vignets2image.py | 4 +- .../modules/execute_example_runner.py | 4 +- .../find_exposures_package/__init__.py | 2 +- .../modules/get_images_package/get_images.py | 24 ++- src/shapepipe/modules/get_images_runner.py | 15 +- .../modules/make_cat_package/make_cat.py | 36 +++- src/shapepipe/modules/make_cat_runner.py | 8 +- src/shapepipe/modules/mask_package/mask.py | 62 ++++-- src/shapepipe/modules/mask_runner.py | 6 +- .../match_external_package/match_external.py | 10 +- .../modules/match_external_runner.py | 8 +- src/shapepipe/modules/mccd_fit_runner.py | 4 +- src/shapepipe/modules/mccd_fit_val_runner.py | 4 +- src/shapepipe/modules/mccd_interp_runner.py | 4 +- .../mccd_package/mccd_interpolation_script.py | 65 +++++-- .../mccd_package/mccd_plot_utilities.py | 184 ++++++++++++++---- .../mccd_package/shapepipe_auxiliary_mccd.py | 55 ++++-- .../modules/mccd_preprocessing_runner.py | 4 +- .../merge_starcat_package/merge_starcat.py | 60 ++++-- src/shapepipe/modules/ngmix_package/ngmix.py | 88 ++++++--- src/shapepipe/modules/ngmix_runner.py | 6 +- .../modules/pastecat_package/pastecat.py | 6 +- src/shapepipe/modules/pastecat_runner.py | 4 +- .../psfex_interp_package/psfex_interp.py | 97 ++++++--- src/shapepipe/modules/psfex_runner.py | 4 +- .../modules/python_example_runner.py | 4 +- .../modules/random_cat_package/random_cat.py | 8 +- src/shapepipe/modules/random_cat_runner.py | 12 +- .../modules/serial_example_runner.py | 4 +- .../modules/setools_package/setools.py | 22 ++- .../sextractor_package/sextractor_script.py | 30 ++- src/shapepipe/modules/sextractor_runner.py | 4 +- .../modules/split_exp_package/split_exp.py | 15 +- src/shapepipe/modules/split_exp_runner.py | 6 +- .../spread_model_package/spread_model.py | 17 +- src/shapepipe/modules/spread_model_runner.py | 6 +- .../vignetmaker_package/vignetmaker.py | 11 +- src/shapepipe/pipeline/args.py | 4 +- src/shapepipe/pipeline/config.py | 3 +- src/shapepipe/pipeline/dependency_handler.py | 21 +- src/shapepipe/pipeline/execute.py | 3 +- src/shapepipe/pipeline/file_handler.py | 51 +++-- src/shapepipe/pipeline/file_io.py | 15 +- src/shapepipe/pipeline/job_handler.py | 16 +- src/shapepipe/pipeline/run_log.py | 3 +- src/shapepipe/pipeline/shared.py | 4 +- src/shapepipe/pipeline/str_handler.py | 23 ++- src/shapepipe/pipeline/timeout.py | 4 +- src/shapepipe/pipeline/worker_handler.py | 12 +- src/shapepipe/run.py | 28 ++- src/shapepipe/tests/test_get_images.py | 8 +- src/shapepipe/utilities/cfis.py | 60 ++++-- src/shapepipe/utilities/file_system.py | 8 +- src/shapepipe/utilities/galaxy.py | 11 +- 62 files changed, 1028 insertions(+), 324 deletions(-) diff --git a/scripts/python/canfar_avail_results.py b/scripts/python/canfar_avail_results.py index 490a0083..62fbf056 100755 --- a/scripts/python/canfar_avail_results.py +++ b/scripts/python/canfar_avail_results.py @@ -83,7 +83,9 @@ def parse_options(p_def): dest="input_path", type="string", default=p_def.input_path, - help="input path, local or vos url, default='{}'".format(p_def.input_path), + help="input path, local or vos url, default='{}'".format( + p_def.input_path + ), ) parser.add_option( "-o", @@ -99,7 +101,9 @@ def parse_options(p_def): dest="psf", type="string", default=p_def.psf, - help="PSF model, one in ['psfex'|'mccd'], default='{}'".format(p_def.psf), + help="PSF model, one in ['psfex'|'mccd'], default='{}'".format( + p_def.psf + ), ) parser.add_option( @@ -125,7 +129,11 @@ def parse_options(p_def): help=f"file extension, default='{p_def.extension}'", ) parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose output", ) options, args = parser.parse_args() @@ -227,7 +235,12 @@ def read_input_files(input_path, verbose=False): def check_results( - ID_files, input_path, result_base_names, n_complete, extension, verbose=False + ID_files, + input_path, + result_base_names, + n_complete, + extension, + verbose=False, ): """Count the number of result files uploaded to vos for each input ID file. @@ -314,7 +327,10 @@ def output_summary(n_found, n_IDs, n_complete): nf = sum(value == n_complete for value in n_found[ID_list].values()) print( "{}: {}/{} ({:.1f}%) complete".format( - os.path.basename(ID_list), nf, n_IDs[ID_list], nf / n_IDs[ID_list] * 100 + os.path.basename(ID_list), + nf, + n_IDs[ID_list], + nf / n_IDs[ID_list] * 100, ) ) diff --git a/scripts/python/canfar_run_analyse.py b/scripts/python/canfar_run_analyse.py index 488b80c7..72da6047 100755 --- a/scripts/python/canfar_run_analyse.py +++ b/scripts/python/canfar_run_analyse.py @@ -88,7 +88,11 @@ def parse_options(p_def): ) parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose output", ) options, args = parser.parse_args() @@ -201,7 +205,10 @@ def get_status(tile_num): for line_err in err_file: mm = re.search("NodeNotFound", line_err) if mm: - status = status + (fail_vos_not_found, "vos file not found") + status = status + ( + fail_vos_not_found, + "vos file not found", + ) break mm = re.search("Empty or corrupt FITS file", line_err) if mm: @@ -216,21 +223,35 @@ def get_status(tile_num): if mm: status = status + (fail_time_out, "vos time out") break - mm = re.search("ERROR:: __str__ returned non-string", line_err) + mm = re.search( + "ERROR:: __str__ returned non-string", line_err + ) if mm: - status = status + (fail_connection, "Connection/HTTP error") + status = status + ( + fail_connection, + "Connection/HTTP error", + ) break mm = re.search("ERROR:: 503 Server Error", line_err) if mm: - status = status + (fail_server, "server connection error") + status = status + ( + fail_server, + "server connection error", + ) break mm = re.search("ERROR:: \[Errno 14\] vos", line_err) if mm: - status = status + (fail_transient, "TransientException") + status = status + ( + fail_transient, + "TransientException", + ) break mm = re.search("Connection aborted", line_err) if mm: - status = status + (fail_vos_no_resp, "vos no response") + status = status + ( + fail_vos_no_resp, + "vos no response", + ) break if len(status) == 2: @@ -301,7 +322,10 @@ def output_failed(output_fail, status): with open(output_fail, "w") as f_out: for tile_num in status.keys(): - if status[tile_num][0] == res_noout or status[tile_num][0] == res_unk: + if ( + status[tile_num][0] == res_noout + or status[tile_num][0] == res_unk + ): print(tile_num, file=f_out) diff --git a/scripts/python/cfis_field_select.py b/scripts/python/cfis_field_select.py index 9692c547..08f79d80 100755 --- a/scripts/python/cfis_field_select.py +++ b/scripts/python/cfis_field_select.py @@ -64,7 +64,9 @@ def get_images_used_in_tiles(images, band, image_type): m = re.search(pattern, temp[3]) if not m: raise cfis.CfisError( - "re match '{}' failed for filename '{}'".format(pattern, temp[3]) + "re match '{}' failed for filename '{}'".format( + pattern, temp[3] + ) ) exp_name = m.group(1) @@ -75,7 +77,9 @@ def get_images_used_in_tiles(images, band, image_type): return exp_list_uniq -def get_coord_at_image(number, band, image_type, images, no_cuts=False, verbose=False): +def get_coord_at_image( + number, band, image_type, images, no_cuts=False, verbose=False +): """Return coordinate of image with given number. Parameters @@ -107,7 +111,9 @@ def get_coord_at_image(number, band, image_type, images, no_cuts=False, verbose= if verbose == True: print( - "Looking for coordinates for tile with numbers ({},{})".format(nix, niy) + "Looking for coordinates for tile with numbers ({},{})".format( + nix, niy + ) ) ra, dec = cfis.get_tile_coord_from_nixy(nix, niy) @@ -123,7 +129,9 @@ def get_coord_at_image(number, band, image_type, images, no_cuts=False, verbose= img_found = img else: - raise cfis.CfisError("Image type '{}' not implemented yet".format(image_type)) + raise cfis.CfisError( + "Image type '{}' not implemented yet".format(image_type) + ) return img_found @@ -319,7 +327,11 @@ def parse_options(p_def): # Monitoring parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose output", ) options, args = parser.parse_args() @@ -353,17 +365,23 @@ def check_options(options): ) if options.image_type != "exposure" and options.no_cuts == True: - raise cfis.CfisError("option '--no_cuts' only possible for image_type=exposure") + raise cfis.CfisError( + "option '--no_cuts' only possible for image_type=exposure" + ) if options.input in [ "{}.txt".format(options.outbase), "{}.pdf", format(options.outbase), ]: - raise cfis.CfisError("Output base same as input, latter will be overwritten!") + raise cfis.CfisError( + "Output base same as input, latter will be overwritten!" + ) if options.input_format not in ["full", "ID_only"]: - raise cfis.CfisError("input_format needs to be one of 'fulll', 'ID_only'") + raise cfis.CfisError( + "input_format needs to be one of 'fulll', 'ID_only'" + ) see_help = "See option '-h' for help." @@ -520,7 +538,9 @@ def run_mode(images, param): elif param.tile: # Search exposures used in input tile(s) - images_found = get_images_used_in_tiles(images, param.band, param.image_type) + images_found = get_images_used_in_tiles( + images, param.band, param.image_type + ) if len(images_found) > 0: for img in images_found: print(img, file=param.fout) @@ -528,7 +548,8 @@ def run_mode(images, param): else: raise cfis.CfisError( - "One of '--coord', '--number', " "'--area', '--tile' needs to be specified" + "One of '--coord', '--number', " + "'--area', '--tile' needs to be specified" ) return ex @@ -582,7 +603,9 @@ def main(argv=None): ) if len(images) == 0: raise cfis.CfisError( - "No corresponding image files found in input '{}'".format(param.input) + "No corresponding image files found in input '{}'".format( + param.input + ) ) # Run diff --git a/scripts/python/create_sample_results.py b/scripts/python/create_sample_results.py index e6e18de3..0e0223ff 100755 --- a/scripts/python/create_sample_results.py +++ b/scripts/python/create_sample_results.py @@ -93,7 +93,11 @@ def parse_options(p_def): # Control parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose output", ) options, args = parser.parse_args() @@ -191,7 +195,9 @@ def read_ID_list(input_ID_path, verbose=False): return input_IDs -def create_links(input_dir, output_dir, input_IDs, result_base_names, verbose=False): +def create_links( + input_dir, output_dir, input_IDs, result_base_names, verbose=False +): """Create symbolic links to result files corresponding to (sub-)sample. Parameters @@ -239,7 +245,11 @@ def create_links(input_dir, output_dir, input_IDs, result_base_names, verbose=Fa if verbose: print("{:5d} links created".format(n_created)) print("{:5d} links existed already".format(n_existed)) - print("{:5d}/{} links available now".format(n_created + n_existed, n_expected)) + print( + "{:5d}/{} links available now".format( + n_created + n_existed, n_expected + ) + ) n_tot = sum(n_total.values()) print("{:5d} as cross-check".format(n_tot)) @@ -285,7 +295,11 @@ def main(argv=None): if os.path.isdir(param.output_dir): if param.verbose: - print("Directory {} already exists, continuing...".format(param.output_dir)) + print( + "Directory {} already exists, continuing...".format( + param.output_dir + ) + ) else: mkdir(param.output_dir) diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py index dce2b390..f990942f 100755 --- a/scripts/python/create_star_cat.py +++ b/scripts/python/create_star_cat.py @@ -105,7 +105,9 @@ def SphereDist(position1, position2, wcs): distance (in degrees) """ - if (type(position1) is not np.ndarray) & (type(position2) is not np.ndarray): + if (type(position1) is not np.ndarray) & ( + type(position2) is not np.ndarray + ): raise ValueError("Positions need to be of type numpy.ndarray") rad2deg = np.pi / 180.0 @@ -192,7 +194,9 @@ def main(input_dir, output_dir, kind): img_shape = (h["NAXIS2"], h["NAXIS1"]) img_center = np.array([img_shape[1] / 2.0, img_shape[0] / 2.0]) wcs_center = w.all_pix2world([img_center], 1)[0] - astropy_center = SkyCoord(ra=wcs_center[0], dec=wcs_center[1], unit="deg") + astropy_center = SkyCoord( + ra=wcs_center[0], dec=wcs_center[1], unit="deg" + ) rad = _get_image_radius(img_center, w) diff --git a/scripts/python/get_number_objects.py b/scripts/python/get_number_objects.py index 87720edb..aaf6bf70 100755 --- a/scripts/python/get_number_objects.py +++ b/scripts/python/get_number_objects.py @@ -116,7 +116,11 @@ def parse_options(p_def): ) parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose output", ) options, args = parser.parse_args() diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py index a652649e..d921095b 100755 --- a/scripts/python/merge_final_cat.py +++ b/scripts/python/merge_final_cat.py @@ -124,7 +124,11 @@ def parse_options(p_def): ) parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose output", ) options, args = parser.parse_args() diff --git a/scripts/python/stats_global.py b/scripts/python/stats_global.py index 5def1ec9..b3a6964b 100755 --- a/scripts/python/stats_global.py +++ b/scripts/python/stats_global.py @@ -101,7 +101,11 @@ def parse_options(p_def): ) parser.add_option( - "-v", "--verbose", dest="verbose", action="store_true", help="verbose output" + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose output", ) options, args = parser.parse_args() @@ -238,7 +242,9 @@ def gather_values(paths, verbose=False): values[key].append(val) else: if verbose: - print("NaN found in file '{}', key '{}'".format(path, key)) + print( + "NaN found in file '{}', key '{}'".format(path, key) + ) if verbose: print("{} keys created".format(len(values))) @@ -390,7 +396,9 @@ def plot_histograms(hists, config=None, output_dir=".", verbose=False): if verbose: print("Creating files '{}.*'".format(file_base)) - plt.savefig("{}/{}.png".format(output_dir, file_base), bbox_inches="tight") + plt.savefig( + "{}/{}.png".format(output_dir, file_base), bbox_inches="tight" + ) np.savetxt( "{}/{}.txt".format(output_dir, file_base), np.transpose([bins, freq]), @@ -426,7 +434,9 @@ def get_config(config_path, verbose=False): print("Reading configuration file '{}'".format(config_path)) if not os.path.exists(config_path): - raise OSError("Configuration file '{}' does not exist".format(config_path)) + raise OSError( + "Configuration file '{}' does not exist".format(config_path) + ) conf = CustomParser() conf.read(config_path) @@ -467,7 +477,9 @@ def main(argv=None): hists = compute_histograms(values, config=config, verbose=param.verbose) if os.path.isfile(param.output_dir): - raise OSError("Output path '{}' is a regular file" "".format(param.output_dir)) + raise OSError( + "Output path '{}' is a regular file" "".format(param.output_dir) + ) if not os.path.isdir(param.output_dir): os.mkdir(param.output_dir) diff --git a/scripts/python/vignets2image.py b/scripts/python/vignets2image.py index e439ce32..60933173 100755 --- a/scripts/python/vignets2image.py +++ b/scripts/python/vignets2image.py @@ -69,7 +69,9 @@ def main(argv=None): image, nx = map_vignet(hdu[2].data["VIGNET"], "float32") print("file = {}, nx = {}".format(input_path, nx)) - fout = io.FITSCatalog(output_path, open_mode=io.BaseCatalog.OpenMode.ReadWrite) + fout = io.FITSCatalog( + output_path, open_mode=io.BaseCatalog.OpenMode.ReadWrite + ) fout.save_as_fits(image, image=True) return 0 diff --git a/src/shapepipe/modules/execute_example_runner.py b/src/shapepipe/modules/execute_example_runner.py index c34ef6c7..2872d608 100644 --- a/src/shapepipe/modules/execute_example_runner.py +++ b/src/shapepipe/modules/execute_example_runner.py @@ -28,7 +28,9 @@ def execute_example_runner( ): """Define The Execute Example Runner.""" command_line = f"head {input_file_list[0]}" - output_file_name = f'{run_dirs["output"]}/head_output{file_number_string}.txt' + output_file_name = ( + f'{run_dirs["output"]}/head_output{file_number_string}.txt' + ) stdout, stderr = execute(command_line) diff --git a/src/shapepipe/modules/find_exposures_package/__init__.py b/src/shapepipe/modules/find_exposures_package/__init__.py index 6bb5d634..ed60925e 100644 --- a/src/shapepipe/modules/find_exposures_package/__init__.py +++ b/src/shapepipe/modules/find_exposures_package/__init__.py @@ -29,7 +29,7 @@ Column number to find exposure in fits header of tile image for the HISTORY string EXP_PREFIX: str - Prefix of exposures + Prefix of exposures """ __all__ = ["find_exposures.py"] diff --git a/src/shapepipe/modules/get_images_package/get_images.py b/src/shapepipe/modules/get_images_package/get_images.py index 52e1d5c2..631a876f 100644 --- a/src/shapepipe/modules/get_images_package/get_images.py +++ b/src/shapepipe/modules/get_images_package/get_images.py @@ -153,7 +153,9 @@ def process(self, input_dir, output_dir): # Get unique number list image_number_list = list(set(flat_list)) - self._w_log.info(f"Number of unique image IDs = {len(image_number_list)}") + self._w_log.info( + f"Number of unique image IDs = {len(image_number_list)}" + ) # Create array to make it compatible with input dir nitem = len(input_dir) @@ -242,7 +244,10 @@ def get_file_list( fbase = re.sub(self._input_numbering, number, in_pattern) ext_final = in_ext - if use_output_file_pattern and self._output_file_pattern[idx] == "*": + if ( + use_output_file_pattern + and self._output_file_pattern[idx] == "*" + ): # retrieve all input files to output dir, do not append # extension fpath = in_path @@ -279,7 +284,9 @@ def retrieve(self, all_inputs, all_outputs_orig, all_outputs_renamed): ) if path: if len(path) == self._n_expected: - self._w_log.info(f"{path[0]} found, skipping download") + self._w_log.info( + f"{path[0]} found, skipping download" + ) continue else: self._w_log.info( @@ -295,7 +302,9 @@ def retrieve(self, all_inputs, all_outputs_orig, all_outputs_renamed): + " downloading images" ) self.retrieve_one( - in_per_type[idx], out_per_type_orig[idx], out_per_type_renamed[idx] + in_per_type[idx], + out_per_type_orig[idx], + out_per_type_renamed[idx], ) def retrieve_one(self, in_path, out_path_orig, out_path_renamed): @@ -337,7 +346,8 @@ def retrieve_one(self, in_path, out_path_orig, out_path_renamed): except Exception: attempt += 1 self._w_log.info( - "Error with command vcp, attempt " + f"{attempt}/{self._n_try}" + "Error with command vcp, attempt " + + f"{attempt}/{self._n_try}" ) sys.argv = None @@ -352,7 +362,9 @@ def retrieve_one(self, in_path, out_path_orig, out_path_renamed): # Get all input file names if INPUT_FILE_PATTERN contains '*' all_src = glob.glob(src) if len(all_src) == 0: - raise IndexError(f"No input file found corresponding to '{src}'") + raise IndexError( + f"No input file found corresponding to '{src}'" + ) dst = out_path_renamed for src in all_src: diff --git a/src/shapepipe/modules/get_images_runner.py b/src/shapepipe/modules/get_images_runner.py index b1a03b15..fb25e0e8 100644 --- a/src/shapepipe/modules/get_images_runner.py +++ b/src/shapepipe/modules/get_images_runner.py @@ -32,11 +32,14 @@ def get_images_runner( retrieve_ok = ["vos", "symlink"] if retrieve_method not in retrieve_ok: raise ValueError( - f"key RETRIEVE={retrieve_method} is invalid, " + f"must be in {retrieve_ok}" + f"key RETRIEVE={retrieve_method} is invalid, " + + f"must be in {retrieve_ok}" ) if config.has_option(module_config_sec, "RETRIEVE_OPTIONS"): - retrieve_options = config.getexpanded(module_config_sec, "RETRIEVE_OPTIONS") + retrieve_options = config.getexpanded( + module_config_sec, "RETRIEVE_OPTIONS" + ) else: retrieve_options = None @@ -49,7 +52,9 @@ def get_images_runner( input_dir = config.getlist(module_config_sec, "INPUT_PATH") input_file_pattern = config.getlist(module_config_sec, "INPUT_FILE_PATTERN") input_file_ext = config.getlist(module_config_sec, "INPUT_FILE_EXT") - output_file_pattern = config.getlist(module_config_sec, "OUTPUT_FILE_PATTERN") + output_file_pattern = config.getlist( + module_config_sec, "OUTPUT_FILE_PATTERN" + ) input_numbering = config.get(module_config_sec, "INPUT_NUMBERING") @@ -60,7 +65,9 @@ def get_images_runner( # Flags to check for already retrieved files if config.has_option(module_config_sec, "CHECK_EXISTING_DIR"): - check_existing_dir = config.getexpanded(module_config_sec, "CHECK_EXISTING_DIR") + check_existing_dir = config.getexpanded( + module_config_sec, "CHECK_EXISTING_DIR" + ) if config.has_option(module_config_sec, "N_EXPECTED"): n_expected = config.getint(module_config_sec, "N_EXPECTED") else: diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 020d9398..e9b246a7 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -96,7 +96,11 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True): cat_size = len(data) tile_id = float( - ".".join(re.split("-", os.path.splitext(os.path.split(sexcat_path)[1])[0])[1:]) + ".".join( + re.split("-", os.path.splitext(os.path.split(sexcat_path)[1])[0])[ + 1: + ] + ) ) tile_id_array = np.ones(cat_size) * tile_id @@ -231,7 +235,10 @@ def _update_dict(self, key_string, value): """ self._output_dict = { **self._output_dict, - **{f"{key_string}{key_end}": np.copy(value) for key_end in self._key_ends}, + **{ + f"{key_string}{key_end}": np.copy(value) + for key_end in self._key_ends + }, } def _add2dict(self, key, value, index=None): @@ -298,7 +305,11 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): self._update_dict(key_str, np.zeros(len(self._obj_id))) for key_str in (f"NGMIX{m}_FLUX_ERR_", f"NGMIX{m}_MAG_ERR_"): self._update_dict(key_str, np.ones(len(self._obj_id)) * -1) - for key_str in (f"NGMIX{m}_ELL_", f"NGMIX{m}_ELL_ERR_", f"NGMIX{m}_ELL_PSFo_"): + for key_str in ( + f"NGMIX{m}_ELL_", + f"NGMIX{m}_ELL_ERR_", + f"NGMIX{m}_ELL_PSFo_", + ): self._update_dict(key_str, np.ones((len(self._obj_id), 2)) * -10.0) self._update_dict( f"NGMIX{m}_T_ERR_", @@ -321,7 +332,10 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): ncf_data = ngmix_cat_file.get_data(key) g = (ncf_data["g1"][ind[0]], ncf_data["g2"][ind[0]]) - g_err = (ncf_data["g1_err"][ind[0]], ncf_data["g2_err"][ind[0]]) + g_err = ( + ncf_data["g1_err"][ind[0]], + ncf_data["g2_err"][ind[0]], + ) self._add2dict(f"NGMIX{m}_ELL_{key}", g, idx) self._add2dict(f"NGMIX{m}_ELL_ERR_{key}", g_err, idx) @@ -437,7 +451,10 @@ def _save_galsim_shapes(self, galsim_cat_path): else: - g = (gcf_data["gal_g1"][ind[0]], gcf_data["gal_g2"][ind[0]]) + g = ( + gcf_data["gal_g1"][ind[0]], + gcf_data["gal_g2"][ind[0]], + ) g_err = ( gcf_data["gal_g1_err"][ind[0]], gcf_data["gal_g2_err"][ind[0]], @@ -458,7 +475,10 @@ def _save_galsim_shapes(self, galsim_cat_path): sigma = gcf_data["gal_sigma"][ind[0]] self._add2dict(f"GALSIM_GAL_SIGMA_{key}", sigma, idx) - psf_g = (gcf_data["psf_g1"][ind[0]], gcf_data["psf_g2"][ind[0]]) + psf_g = ( + gcf_data["psf_g1"][ind[0]], + gcf_data["psf_g2"][ind[0]], + ) psf_sigma = gcf_data["psf_sigma"][ind[0]] self._add2dict(f"GALSIM_PSF_ELL_{key}", psf_g, idx) self._add2dict( @@ -537,7 +557,9 @@ def _save_psf_data(self, galaxy_psf_path): ) self._add2dict(f"PSF_ELL_{epoch + 1}", e_psf, idx) - psf_fwhm = galaxy.sigma_to_fwhm(gpc_data["SHAPES"]["SIGMA_PSF_HSM"]) + psf_fwhm = galaxy.sigma_to_fwhm( + gpc_data["SHAPES"]["SIGMA_PSF_HSM"] + ) self._add2dict(f"PSF_FWHM_{epoch + 1}", psf_fwhm, idx) flag_psf = gpc_data["SHAPES"]["FLAG_PSF_HSM"] diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index 2b171b57..f36209f4 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -65,7 +65,9 @@ def make_cat_runner( ) for shape_type in shape_type_list: if shape_type.lower() not in ["ngmix", "galsim"]: - raise ValueError("SHAPE_MEASUREMENT_TYPE must be in [ngmix, galsim]") + raise ValueError( + "SHAPE_MEASUREMENT_TYPE must be in [ngmix, galsim]" + ) # Fetch PSF data option if config.has_option(module_config_sec, "SAVE_PSF_DATA"): @@ -98,7 +100,9 @@ def make_cat_runner( w_log.info("Save shape measurement data") for shape_type in shape_type_list: w_log.info(f"Save {shape_type.lower()} data") - cat_path = shape2_cat_path if shape_type == "galsim" else shape1_cat_path + cat_path = ( + shape2_cat_path if shape_type == "galsim" else shape1_cat_path + ) sc_inst.process(shape_type.lower(), cat_path) if save_psf: sc_inst.process("psf", galaxy_psf_path) diff --git a/src/shapepipe/modules/mask_package/mask.py b/src/shapepipe/modules/mask_package/mask.py index 8dcbdd59..e4bcca53 100644 --- a/src/shapepipe/modules/mask_package/mask.py +++ b/src/shapepipe/modules/mask_package/mask.py @@ -146,7 +146,9 @@ def _get_config(self): } if conf.has_option("PROGRAM_PATH", "WW_PATH"): - self._config["PATH"]["WW"] = conf.getexpanded("PROGRAM_PATH", "WW_PATH") + self._config["PATH"]["WW"] = conf.getexpanded( + "PROGRAM_PATH", "WW_PATH" + ) else: self._config["PATH"]["WW"] = "ww" self._config["PATH"]["WW_configfile"] = conf.getexpanded( @@ -253,7 +255,9 @@ def _get_config(self): self._config["MD"]["thresh_remove"] = conf.getfloat( "MD_PARAMETERS", "MD_THRESH_REMOVE" ) - self._config["MD"]["remove"] = conf.getboolean("MD_PARAMETERS", "MD_REMOVE") + self._config["MD"]["remove"] = conf.getboolean( + "MD_PARAMETERS", "MD_REMOVE" + ) def _set_image_coordinates(self): """Set Image Coordinates. @@ -411,11 +415,15 @@ def make_mask(self): general_stdout += f"\n\nrm reg file\n{self._rm_reg_stdout}" if self._rm_reg_stderr != "": general_stderr += f"\n\nrm reg file\n{self._rm_reg_stderr}" - if hasattr(self, "_rm_fits1_stderr") or hasattr(self, "_rm_fits1_stdout"): + if hasattr(self, "_rm_fits1_stderr") or hasattr( + self, "_rm_fits1_stdout" + ): general_stdout += f"\n\nrm fits1 file\n{self._rm_fits1_stdout}" if self._rm_fits1_stderr != "": general_stderr += f"\n\nrm fits1 file\n{self._rm_fits1_stderr}" - if hasattr(self, "_rm_fits2_stderr") or hasattr(self, "_rm_fits2_stdout"): + if hasattr(self, "_rm_fits2_stderr") or hasattr( + self, "_rm_fits2_stdout" + ): general_stdout += f"\n\nrm fits2 file\n{self._rm_fits2_stdout}" if self._rm_fits2_stderr != "": general_stderr += f"\n\nrm fits2 file\n{self._rm_fits2_stderr}" @@ -561,7 +569,9 @@ def mask_dso( """ if size_plus < 0: - raise ValueError("deep-sky mask size increase variable cannot be negative") + raise ValueError( + "deep-sky mask size increase variable cannot be negative" + ) if cat_path is None: raise ValueError("Path to deep-sky object catalogue not provided") @@ -647,7 +657,9 @@ def mask_dso( # The following accounts for deep-sky centers outside of image, # without creating masks for coordinates out of range y_c, x_c = np.ogrid[0:ny, 0:nx] - mask_tmp = (x_c - m_center[0]) ** 2 + (y_c - m_center[1]) ** 2 <= r_pix**2 + mask_tmp = (x_c - m_center[0]) ** 2 + ( + y_c - m_center[1] + ) ** 2 <= r_pix**2 flag[mask_tmp] = flag_value @@ -714,7 +726,10 @@ def sphere_dist(self, position1, position2): If input positions are not Numpy arrays """ - if type(position1) is not np.ndarray or type(position2) is not np.ndarray: + if ( + type(position1) is not np.ndarray + or type(position2) is not np.ndarray + ): raise ValueError("Object coordinates need to be a numpy.ndarray") p1 = (np.pi / 180.0) * np.hstack( @@ -759,13 +774,17 @@ def _get_image_radius(self, center=None): """ if center is None: - return self.sphere_dist(self._fieldcenter["pix"], np.zeros(2)) / 60.0 + return ( + self.sphere_dist(self._fieldcenter["pix"], np.zeros(2)) / 60.0 + ) else: if isinstance(center, np.ndarray): return self.sphere_dist(center, np.zeros(2)) / 60.0 else: - raise TypeError("Image center coordinates has to be a numpy.ndarray") + raise TypeError( + "Image center coordinates has to be a numpy.ndarray" + ) def _make_star_cat(self, CDSclient_output): """Make Star Catalogue. @@ -867,7 +886,9 @@ def _create_mask( else: reg = self._config[types]["reg_file"] - mask_model = np.loadtxt(self._config[types]["maskmodel_path"]).transpose() + mask_model = np.loadtxt( + self._config[types]["maskmodel_path"] + ).transpose() mask_reg = open(reg, "w") stars_used = [[], [], []] @@ -1000,7 +1021,10 @@ def _exec_WW(self, types="HALO"): elif types == "ALL": default_reg = [ - (f'{self._config["PATH"]["temp_dir"]}' + f"halo{self._img_number}.reg"), + ( + f'{self._config["PATH"]["temp_dir"]}' + + f"halo{self._img_number}.reg" + ), ( f'{self._config["PATH"]["temp_dir"]}' + f"spike{self._img_number}.reg" @@ -1015,8 +1039,12 @@ def _exec_WW(self, types="HALO"): reg = default_reg for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) + if not ( + file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx]) + ): + raise ( + file_io.BaseCatalogue.CatalogFileNotFound(reg[idx]) + ) cmd = ( f'{self._config["PATH"]["WW"]} ' @@ -1041,8 +1069,12 @@ def _exec_WW(self, types="HALO"): ] for idx in range(2): - if not (file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx])): - raise (file_io.BaseCatalogue.CatalogFileNotFound(reg[idx])) + if not ( + file_io.BaseCatalogue(reg[idx])._file_exists(reg[idx]) + ): + raise ( + file_io.BaseCatalogue.CatalogFileNotFound(reg[idx]) + ) cmd = ( f'{self._config["PATH"]["WW"]} ' diff --git a/src/shapepipe/modules/mask_runner.py b/src/shapepipe/modules/mask_runner.py index 55721c3a..7d004262 100644 --- a/src/shapepipe/modules/mask_runner.py +++ b/src/shapepipe/modules/mask_runner.py @@ -52,9 +52,9 @@ def mask_runner( # Set options for 4 inputs elif n_inputs == 4: - if config.getboolean(module_config_sec, "USE_EXT_FLAG") and config.getboolean( - module_config_sec, "USE_EXT_STAR" - ): + if config.getboolean( + module_config_sec, "USE_EXT_FLAG" + ) and config.getboolean(module_config_sec, "USE_EXT_STAR"): ext_flag_name = input_file_list[2] ext_star_cat = input_file_list[3] else: diff --git a/src/shapepipe/modules/match_external_package/match_external.py b/src/shapepipe/modules/match_external_package/match_external.py index 06d6e994..218a231e 100644 --- a/src/shapepipe/modules/match_external_package/match_external.py +++ b/src/shapepipe/modules/match_external_package/match_external.py @@ -206,7 +206,11 @@ def process(self): # Get indices in internal and external catalogues of pair-wise # matches w = np.array( - [(idx, ide) for (idx, ide) in enumerate(indices) if indices_close[idx]] + [ + (idx, ide) + for (idx, ide) in enumerate(indices) + if indices_close[idx] + ] ) id_sub = w[:, 0] id_ext_sub = w[:, 1] @@ -221,7 +225,9 @@ def process(self): id_data = id_sub id_ext = id_ext_sub - self._w_log.info(f"{len(id_sub)} objects matched out of {len(indices)}.") + self._w_log.info( + f"{len(id_sub)} objects matched out of {len(indices)}." + ) # Copy matched objects from internal catalogue to output data matched = {} diff --git a/src/shapepipe/modules/match_external_runner.py b/src/shapepipe/modules/match_external_runner.py index c08266a3..4df5ac0c 100644 --- a/src/shapepipe/modules/match_external_runner.py +++ b/src/shapepipe/modules/match_external_runner.py @@ -41,7 +41,9 @@ def match_external_runner( mode = config.get(module_config_sec, "MODE") valid_modes = ("CLASSIC", "MULTI-EPOCH") if mode not in valid_modes: - raise ValueError(f"mode '{mode}' is invalid, must be one of {valid_modes}.") + raise ValueError( + f"mode '{mode}' is invalid, must be one of {valid_modes}." + ) # External data external_cat_path = config.getexpanded( @@ -89,7 +91,9 @@ def match_external_runner( # Set output file path file_ext = "fits" - output_path = f'{run_dirs["output"]}/{prefix}{file_number_string}.' + f"{file_ext}" + output_path = ( + f'{run_dirs["output"]}/{prefix}{file_number_string}.' + f"{file_ext}" + ) # Create instance of MatchCats match_cats_inst = MatchCats( diff --git a/src/shapepipe/modules/mccd_fit_runner.py b/src/shapepipe/modules/mccd_fit_runner.py index c4d8af10..58e36723 100644 --- a/src/shapepipe/modules/mccd_fit_runner.py +++ b/src/shapepipe/modules/mccd_fit_runner.py @@ -56,7 +56,9 @@ def mccd_fit_runner( ) else: - raise ValueError('mccd_fit_runner should be called when the MODE is "FIT".') + raise ValueError( + 'mccd_fit_runner should be called when the MODE is "FIT".' + ) # No return objects return None, None diff --git a/src/shapepipe/modules/mccd_fit_val_runner.py b/src/shapepipe/modules/mccd_fit_val_runner.py index 73f04081..26623c2a 100644 --- a/src/shapepipe/modules/mccd_fit_val_runner.py +++ b/src/shapepipe/modules/mccd_fit_val_runner.py @@ -61,7 +61,9 @@ def mccd_fit_val_runner( ) # Fitted model is found in the output directory - mccd_model_path = output_dir + fit_saving_name + file_number_string + ".npy" + mccd_model_path = ( + output_dir + fit_saving_name + file_number_string + ".npy" + ) aux_mccd.mccd_validation_pipeline( teststar_path=teststar_path, diff --git a/src/shapepipe/modules/mccd_interp_runner.py b/src/shapepipe/modules/mccd_interp_runner.py index 58525bf8..1428e44d 100644 --- a/src/shapepipe/modules/mccd_interp_runner.py +++ b/src/shapepipe/modules/mccd_interp_runner.py @@ -10,7 +10,9 @@ from shapepipe.pipeline.run_log import get_last_dir -from shapepipe.modules.mccd_package import mccd_interpolation_script as mccd_interp +from shapepipe.modules.mccd_package import ( + mccd_interpolation_script as mccd_interp, +) from shapepipe.modules.mccd_package import shapepipe_auxiliary_mccd as aux_mccd from shapepipe.modules.module_decorator import module_runner diff --git a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py index 8bd35da9..61e3bc95 100644 --- a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py +++ b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py @@ -213,7 +213,8 @@ def _get_psfshapes(self): raise ImportError("Galsim is required to get shapes information") psf_moms = [ - hsm.FindAdaptiveMom(Image(psf), strict=False) for psf in self.interp_PSFs + hsm.FindAdaptiveMom(Image(psf), strict=False) + for psf in self.interp_PSFs ] self.psf_shapes = np.array( @@ -306,9 +307,15 @@ def _get_psfexcatdict(self, psfex_cat_path): psfex_cat.open() psfex_cat_dict = {} - psfex_cat_dict["SOURCE_NUMBER"] = np.copy(psfex_cat.get_data()["SOURCE_NUMBER"]) - psfex_cat_dict["DELTAX_IMAGE"] = np.copy(psfex_cat.get_data()["DELTAX_IMAGE"]) - psfex_cat_dict["DELTAY_IMAGE"] = np.copy(psfex_cat.get_data()["DELTAY_IMAGE"]) + psfex_cat_dict["SOURCE_NUMBER"] = np.copy( + psfex_cat.get_data()["SOURCE_NUMBER"] + ) + psfex_cat_dict["DELTAX_IMAGE"] = np.copy( + psfex_cat.get_data()["DELTAX_IMAGE"] + ) + psfex_cat_dict["DELTAY_IMAGE"] = np.copy( + psfex_cat.get_data()["DELTAY_IMAGE"] + ) psfex_cat_dict["CHI2_PSF"] = np.copy(psfex_cat.get_data()["CHI2_PSF"]) return psfex_cat_dict @@ -408,7 +415,9 @@ def _interpolate_me(self): n_epoch = np.copy(cat.get_data()[key_ne]) list_ext_name = cat.get_ext_name() - hdu_ind = [i for i in range(len(list_ext_name)) if "EPOCH" in list_ext_name[i]] + hdu_ind = [ + i for i in range(len(list_ext_name)) if "EPOCH" in list_ext_name[i] + ] final_list = [] for hdu_index in hdu_ind: @@ -437,7 +446,9 @@ def _interpolate_me(self): obj_id = all_id[ind_obj] gal_pos = np.array( self._f_wcs_file[exp_name][ccd]["WCS"].all_world2pix( - self.gal_pos[:, 0][ind_obj], self.gal_pos[:, 1][ind_obj], 0 + self.gal_pos[:, 0][ind_obj], + self.gal_pos[:, 1][ind_obj], + 0, ) ).T @@ -455,7 +466,9 @@ def _interpolate_me(self): ) continue - if isinstance(self.interp_PSFs, str) and (self.interp_PSFs == BAD_CHI2): + if isinstance(self.interp_PSFs, str) and ( + self.interp_PSFs == BAD_CHI2 + ): self._w_log.info( f"Bad chi2 for the psf model in the ccd {ccd} of the" + f" exposure {exp_name}. Object inside this ccd" @@ -476,7 +489,9 @@ def _interpolate_me(self): if array_psf is None: array_psf = np.copy(self.interp_PSFs) else: - array_psf = np.concatenate((array_psf, np.copy(self.interp_PSFs))) + array_psf = np.concatenate( + (array_psf, np.copy(self.interp_PSFs)) + ) if array_id is None: array_id = np.copy(obj_id) @@ -500,9 +515,13 @@ def _interpolate_me(self): if array_exp_name is None: array_exp_name = exp_name_tmp else: - array_exp_name = np.concatenate((array_exp_name, exp_name_tmp)) + array_exp_name = np.concatenate( + (array_exp_name, exp_name_tmp) + ) - final_list.append([array_id, array_psf, array_shape, array_exp_name]) + final_list.append( + [array_id, array_psf, array_shape, array_exp_name] + ) self._f_wcs_file.close() cat.close() @@ -515,15 +534,23 @@ def _interpolate_me(self): where_res = np.where(final_list[j][0] == id_tmp)[0] if len(where_res) != 0: output_dict[id_tmp][final_list[j][3][where_res[0]]] = {} - output_dict[id_tmp][final_list[j][3][where_res[0]]]["VIGNET"] = ( - final_list[j][1][where_res[0]] - ) + output_dict[id_tmp][final_list[j][3][where_res[0]]][ + "VIGNET" + ] = final_list[j][1][where_res[0]] if self._compute_shape: shape_dict = {} - shape_dict["E1_PSF_HSM"] = final_list[j][2][where_res[0]][0] - shape_dict["E2_PSF_HSM"] = final_list[j][2][where_res[0]][1] - shape_dict["SIGMA_PSF_HSM"] = final_list[j][2][where_res[0]][2] - shape_dict["FLAG_PSF_HSM"] = final_list[j][2][where_res[0]][3] + shape_dict["E1_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][0] + shape_dict["E2_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][1] + shape_dict["SIGMA_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][2] + shape_dict["FLAG_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][3] output_dict[id_tmp][final_list[j][3][where_res[0]]][ "SHAPES" ] = shape_dict @@ -546,7 +573,9 @@ def _write_output_me(self, output_dict): """ # np.save(self._output_path+self._img_number, output_dict) - output_file = SqliteDict(self._output_path + self._img_number + ".sqlite") + output_file = SqliteDict( + self._output_path + self._img_number + ".sqlite" + ) for i in output_dict.keys(): output_file[str(i)] = output_dict[i] output_file.commit() diff --git a/src/shapepipe/modules/mccd_package/mccd_plot_utilities.py b/src/shapepipe/modules/mccd_package/mccd_plot_utilities.py index 069b0ae6..5a66ec8d 100644 --- a/src/shapepipe/modules/mccd_package/mccd_plot_utilities.py +++ b/src/shapepipe/modules/mccd_package/mccd_plot_utilities.py @@ -161,7 +161,10 @@ def mean_shapes_plot( """ # colorbar amplitude if wind is None: - vmax = max(np.nanmax(ccd_maps), np.abs(np.nanmin(ccd_maps))) * colorbar_ampl + vmax = ( + max(np.nanmax(ccd_maps), np.abs(np.nanmin(ccd_maps))) + * colorbar_ampl + ) vmin = -vmax * colorbar_ampl else: vmin, vmax = wind[0] * colorbar_ampl, wind[1] * colorbar_ampl @@ -326,9 +329,15 @@ def plot_meanshapes( all_Y = all_Y[~bad_stars] flagmask = flagmask[~bad_stars] - e1_res_rmse = np.sqrt(np.mean((all_star_shapes[0, :] - all_psf_shapes[0, :]) ** 2)) - e2_res_rmse = np.sqrt(np.mean((all_star_shapes[1, :] - all_psf_shapes[1, :]) ** 2)) - R2_res_rmse = np.sqrt(np.mean((all_star_shapes[2, :] - all_psf_shapes[2, :]) ** 2)) + e1_res_rmse = np.sqrt( + np.mean((all_star_shapes[0, :] - all_psf_shapes[0, :]) ** 2) + ) + e2_res_rmse = np.sqrt( + np.mean((all_star_shapes[1, :] - all_psf_shapes[1, :]) ** 2) + ) + R2_res_rmse = np.sqrt( + np.mean((all_star_shapes[2, :] - all_psf_shapes[2, :]) ** 2) + ) w_log.info(f"TOTAL e1 residual RMSE: {e1_res_rmse:.6e}\n") w_log.info(f"TOTAL e2 residual RMSE: {e2_res_rmse:.6e}\n") w_log.info(f"TOTAL R2 residual RMSE: {R2_res_rmse:.6e}\n") @@ -341,7 +350,9 @@ def plot_meanshapes( # handle different input catalogue types if psf_model_type == "mccd": - ccd_mask = ((all_CCDs.astype(int) == ccd_nb) * flagmask).astype(bool) + ccd_mask = ((all_CCDs.astype(int) == ccd_nb) * flagmask).astype( + bool + ) # Calculate shift to go from global coordinates to local # coordinates @@ -376,8 +387,12 @@ def plot_meanshapes( for xb in range(nb_pixel[0]): for yb in range(nb_pixel[1]): - bin_star_shapes = star_shapes[:, (xbins == xb + 1) * (ybins == yb + 1)] - bin_psf_shapes = psf_shapes[:, (xbins == xb + 1) * (ybins == yb + 1)] + bin_star_shapes = star_shapes[ + :, (xbins == xb + 1) * (ybins == yb + 1) + ] + bin_psf_shapes = psf_shapes[ + :, (xbins == xb + 1) * (ybins == yb + 1) + ] ccd_map[0, :3, xb, yb] = np.mean(bin_star_shapes, axis=1) ccd_map[1, :3, xb, yb] = np.mean(bin_psf_shapes, axis=1) ccd_map[:, 3, xb, yb] = bin_star_shapes.shape[1] @@ -388,7 +403,8 @@ def plot_meanshapes( vmax = max_e else: vmax = max( - np.nanmax(ccd_maps[:, :, 0]), np.abs(np.nanmin(ccd_maps[:, :, 0])) + np.nanmax(ccd_maps[:, :, 0]), + np.abs(np.nanmin(ccd_maps[:, :, 0])), ) vmin = -vmax wind = [vmin, vmax] @@ -396,13 +412,17 @@ def plot_meanshapes( f"e_1 (stars), std={np.nanstd(ccd_maps[:, 0, 0]):.5e}\n" + f"vmax={np.nanmax(abs(ccd_maps[:, 0, 0])):.4e}" ) - mean_shapes_plot(ccd_maps[:, 0, 0], output_path + "e1s", title, wind=wind) + mean_shapes_plot( + ccd_maps[:, 0, 0], output_path + "e1s", title, wind=wind + ) title = ( f"e_1 (model), std={np.nanstd(ccd_maps[:, 1, 0]):.5e}\n" + f"vmax={np.nanmax(abs(ccd_maps[:, 1, 0])):.4e}" ) - mean_shapes_plot(ccd_maps[:, 1, 0], output_path + "e1m", title, wind=wind) + mean_shapes_plot( + ccd_maps[:, 1, 0], output_path + "e1m", title, wind=wind + ) if auto_colorbar: wind = None @@ -433,7 +453,8 @@ def plot_meanshapes( vmax = max_e else: vmax = max( - np.nanmax(ccd_maps[:, :, 1]), np.abs(np.nanmin(ccd_maps[:, :, 1])) + np.nanmax(ccd_maps[:, :, 1]), + np.abs(np.nanmin(ccd_maps[:, :, 1])), ) vmin = -vmax wind = [vmin, vmax] @@ -441,12 +462,16 @@ def plot_meanshapes( f"e_2 (stars), std={np.nanstd(ccd_maps[:, 0, 1]):.5e}\n" + f"vmax={np.nanmax(abs(ccd_maps[:, 0, 1])):.4e}" ) - mean_shapes_plot(ccd_maps[:, 0, 1], output_path + "e2s", title, wind=wind) + mean_shapes_plot( + ccd_maps[:, 0, 1], output_path + "e2s", title, wind=wind + ) title = ( f"e_2 (model), std={np.nanstd(ccd_maps[:, 1, 1]):.5e}\n" + f"vmax={np.nanmax(abs(ccd_maps[:, 1, 1])):.4e}" ) - mean_shapes_plot(ccd_maps[:, 1, 1], output_path + "e2m", title, wind=wind) + mean_shapes_plot( + ccd_maps[:, 1, 1], output_path + "e2m", title, wind=wind + ) if auto_colorbar: wind = None @@ -490,21 +515,32 @@ def plot_meanshapes( + f"vmax={np.nanmax(abs(ccd_maps[:, 0, 2])):.4e}" ) mean_shapes_plot( - ccd_maps[:, 0, 2], output_path + "R2s", title, wind=wind, cmap="Reds" + ccd_maps[:, 0, 2], + output_path + "R2s", + title, + wind=wind, + cmap="Reds", ) title = ( f"R_2 (model), std={np.nanstd(ccd_maps[:, 1, 2]):.5e}\n" + f"vmax={np.nanmax(abs(ccd_maps[:, 1, 2])):.4e}" ) mean_shapes_plot( - ccd_maps[:, 1, 2], output_path + "R2m", title, wind=wind, cmap="Reds" + ccd_maps[:, 1, 2], + output_path + "R2m", + title, + wind=wind, + cmap="Reds", ) if auto_colorbar: wind = [ 0, np.nanmax( - np.abs((ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) / ccd_maps[:, 0, 2]) + np.abs( + (ccd_maps[:, 0, 2] - ccd_maps[:, 1, 2]) + / ccd_maps[:, 0, 2] + ) ), ] colorbar_ampl = 1.0 @@ -542,7 +578,11 @@ def plot_meanshapes( wind = (0, np.max(ccd_maps[:, 0, 3])) title = f"Number of stars\nTotal={np.nansum(ccd_maps[:, 0, 3]):.0f}" mean_shapes_plot( - ccd_maps[:, 0, 3], f"{output_path}nstar", title, wind=wind, cmap="magma" + ccd_maps[:, 0, 3], + f"{output_path}nstar", + title, + wind=wind, + cmap="magma", ) # Histograms @@ -620,10 +660,18 @@ def plot_meanshapes( mean_R2 = np.mean(all_star_shapes[2, :]) wind = [mean_R2 - 4, mean_R2 + 4] plt.hist( - all_star_shapes[2, :], bins=hist_bins, range=wind, label="stars", alpha=0.5 + all_star_shapes[2, :], + bins=hist_bins, + range=wind, + label="stars", + alpha=0.5, ) plt.hist( - all_psf_shapes[2, :], bins=hist_bins, range=wind, label="PSFs", alpha=0.5 + all_psf_shapes[2, :], + bins=hist_bins, + range=wind, + label="PSFs", + alpha=0.5, ) plt.legend(loc="best", fontsize=16) plt.title("R2", fontsize=24) @@ -631,9 +679,9 @@ def plot_meanshapes( plt.close() plt.figure(figsize=(12, 6), dpi=300) - data_hist = (all_star_shapes[2, :] - all_psf_shapes[2, :]) / all_star_shapes[ - 2, : - ] + data_hist = ( + all_star_shapes[2, :] - all_psf_shapes[2, :] + ) / all_star_shapes[2, :] plt.hist( data_hist, bins=hist_bins, @@ -788,7 +836,9 @@ def neg_dash( if current_sign > 0: plt.errorbar(x, y, yerr=yerr, linestyle="-", label=lab, **errbkwargs) else: - plt.errorbar(x, np.abs(y), yerr=yerr, linestyle="--", label=lab, **errbkwargs) + plt.errorbar( + x, np.abs(y), yerr=yerr, linestyle="--", label=lab, **errbkwargs + ) if semilogx: plt.xscale("log") if semilogy: @@ -809,7 +859,9 @@ class new_BaseCorrelationFunctionSysTest(BaseCorrelationFunctionSysTest): """ - def make_catalogue(self, data, config=None, use_as_k=None, use_chip_coords=False): + def make_catalogue( + self, data, config=None, use_as_k=None, use_chip_coords=False + ): """Make Catalogue. Parameters @@ -1011,7 +1063,13 @@ def __call__( ) if random is not None: new_random = np.rec.fromarrays( - [random["ra"], random["dec"], random["g1"], random["g2"], random["w"]], + [ + random["ra"], + random["dec"], + random["g1"], + random["g2"], + random["w"], + ], names=["ra", "dec", "g1", "g2", "w"], ) @@ -1033,7 +1091,13 @@ def __call__( else: new_random2 = random2 return self.getCF( - "gg", new_data, new_data2, new_random, new_random2, config=config, **kwargs + "gg", + new_data, + new_data2, + new_random, + new_random2, + config=config, + **kwargs, ) @@ -1081,8 +1145,12 @@ def __call__( [ data["ra"], data["dec"], - data["g1"] * (data["sigma"] - data["psf_sigma"]) / data["sigma"], - data["g2"] * (data["sigma"] - data["psf_sigma"]) / data["sigma"], + data["g1"] + * (data["sigma"] - data["psf_sigma"]) + / data["sigma"], + data["g2"] + * (data["sigma"] - data["psf_sigma"]) + / data["sigma"], data["w"], ], names=["ra", "dec", "g1", "g2", "w"], @@ -1143,7 +1211,13 @@ def __call__( new_random2 = random2 return self.getCF( - "gg", new_data, new_data2, new_random, new_random2, config=config, **kwargs + "gg", + new_data, + new_data2, + new_random, + new_random2, + config=config, + **kwargs, ) @@ -1205,8 +1279,12 @@ def __call__( [ data2["ra"], data2["dec"], - data2["g1"] * (data2["sigma"] - data2["psf_sigma"]) / data2["sigma"], - data2["g2"] * (data2["sigma"] - data2["psf_sigma"]) / data2["sigma"], + data2["g1"] + * (data2["sigma"] - data2["psf_sigma"]) + / data2["sigma"], + data2["g2"] + * (data2["sigma"] - data2["psf_sigma"]) + / data2["sigma"], data2["w"], ], names=["ra", "dec", "g1", "g2", "w"], @@ -1244,7 +1322,13 @@ def __call__( else: new_random2 = random2 return self.getCF( - "gg", new_data, new_data2, new_random, new_random2, config=config, **kwargs + "gg", + new_data, + new_data2, + new_random, + new_random2, + config=config, + **kwargs, ) @@ -1299,8 +1383,12 @@ def __call__( [ data2["ra"], data2["dec"], - data2["g1"] * (data2["sigma"] - data2["psf_sigma"]) / data2["sigma"], - data2["g2"] * (data2["sigma"] - data2["psf_sigma"]) / data2["sigma"], + data2["g1"] + * (data2["sigma"] - data2["psf_sigma"]) + / data2["sigma"], + data2["g2"] + * (data2["sigma"] - data2["psf_sigma"]) + / data2["sigma"], data2["w"], ], names=["ra", "dec", "g1", "g2", "w"], @@ -1308,7 +1396,13 @@ def __call__( if random is not None: new_random = np.rec.fromarrays( - [random["ra"], random["dec"], random["g1"], random["g2"], random["w"]], + [ + random["ra"], + random["dec"], + random["g1"], + random["g2"], + random["w"], + ], names=["ra", "dec", "g1", "g2", "w"], ) else: @@ -1334,7 +1428,13 @@ def __call__( else: new_random2 = random2 return self.getCF( - "gg", new_data, new_data2, new_random, new_random2, config=config, **kwargs + "gg", + new_data, + new_data2, + new_random, + new_random2, + config=config, + **kwargs, ) @@ -1390,7 +1490,17 @@ def rho_stats( starcat[hdu_no].data["E2_PSF_HSM"], starcat[hdu_no].data["SIGMA_PSF_HSM"] ** 2, ], - names=["w", "ra", "dec", "g1", "g2", "sigma", "psf_g1", "psf_g2", "psf_sigma"], + names=[ + "w", + "ra", + "dec", + "g1", + "g2", + "sigma", + "psf_g1", + "psf_g2", + "psf_sigma", + ], ) # TreeCorr config: diff --git a/src/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py b/src/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py index 8270c4dd..44b7cf55 100644 --- a/src/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py +++ b/src/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py @@ -105,22 +105,35 @@ def print_fun(x): for it in range(catalog_ids.shape[0]): # For each observation position catalog_id = catalog_ids[it] - star_list, pos_list, mask_list, ccd_list, SNR_list, RA_list, DEC_list = ( - mccd_inputs.get_inputs(catalog_id) - ) - - star_list, pos_list, mask_list, ccd_list, SNR_list, RA_list, DEC_list, _ = ( - mccd_inputs.outlier_rejection( - star_list, - pos_list, - mask_list, - ccd_list, - SNR_list, - RA_list, - DEC_list, - shape_std_max=outlier_std_max, - print_fun=print_fun, - ) + ( + star_list, + pos_list, + mask_list, + ccd_list, + SNR_list, + RA_list, + DEC_list, + ) = mccd_inputs.get_inputs(catalog_id) + + ( + star_list, + pos_list, + mask_list, + ccd_list, + SNR_list, + RA_list, + DEC_list, + _, + ) = mccd_inputs.outlier_rejection( + star_list, + pos_list, + mask_list, + ccd_list, + SNR_list, + RA_list, + DEC_list, + shape_std_max=outlier_std_max, + print_fun=print_fun, ) mccd_star_list = [] @@ -166,7 +179,9 @@ def print_fun(x): # If the list is not empty # Concatenate, as fits can't handle list of numpy arrays and # turn into reg format - mccd_stars = mccd.utils.reg_format(np.concatenate(mccd_star_list, axis=2)) + mccd_stars = mccd.utils.reg_format( + np.concatenate(mccd_star_list, axis=2) + ) mccd_poss = np.concatenate(mccd_pos_list, axis=0) mccd_ccds = np.concatenate(mccd_ccd_list, axis=0) @@ -205,7 +220,11 @@ def print_fun(x): } saving_path = ( - output_path + save_name + separator + catalog_id + save_extension + output_path + + save_name + + separator + + catalog_id + + save_extension ) mccd.mccd_utils.save_to_fits(train_dic, saving_path) diff --git a/src/shapepipe/modules/mccd_preprocessing_runner.py b/src/shapepipe/modules/mccd_preprocessing_runner.py index a891e6c0..f15287cf 100644 --- a/src/shapepipe/modules/mccd_preprocessing_runner.py +++ b/src/shapepipe/modules/mccd_preprocessing_runner.py @@ -66,7 +66,9 @@ def mccd_preprocessing_runner( min_n_stars_list = [1] else: - raise ValueError("MODE should be in ['FIT', FIT_VALIDATION', 'VALIDATION'].") + raise ValueError( + "MODE should be in ['FIT', FIT_VALIDATION', 'VALIDATION']." + ) # Use the outfile from the pipeline and ignore the output directory from # the MCCD config file diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index d014ab7f..b5869433 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -39,7 +39,13 @@ class MergeStarCatMCCD(object): """ def __init__( - self, input_file_list, output_dir, w_log, stamp_size=51, rad=10, hdu_table=1 + self, + input_file_list, + output_dir, + w_log, + stamp_size=51, + rad=10, + hdu_table=1, ): self._input_file_list = input_file_list @@ -105,7 +111,9 @@ def rmse_calc_2(values, sizes): Root mean square error """ - rmse = np.sqrt(np.nansum(np.array(values) ** 2) / np.nansum(np.array(sizes))) + rmse = np.sqrt( + np.nansum(np.array(values) ** 2) / np.nansum(np.array(sizes)) + ) return rmse @@ -268,7 +276,9 @@ def process(self): (psfs_norm_vals != 0), (stars_norm_vals != 0) ) # Calculate the filtered mse calculation - pix_filt_val = np.sum((stars[non_zero_elems] - psfs[non_zero_elems]) ** 2) + pix_filt_val = np.sum( + (stars[non_zero_elems] - psfs[non_zero_elems]) ** 2 + ) # Calculate the normalized (& filtered) mse calculation stars_norm_vals = stars_norm_vals[non_zero_elems].reshape(-1, 1, 1) psfs_norm_vals = psfs_norm_vals[non_zero_elems].reshape(-1, 1, 1) @@ -303,8 +313,12 @@ def process(self): model_var_size.append(model_var_val.size) # positions - x += list(starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 0]) - y += list(starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 1]) + x += list( + starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 0] + ) + y += list( + starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 1] + ) # RA and DEC positions try: @@ -329,16 +343,28 @@ def process(self): ) # shapes (convert sigmas to R^2) - g1_psf += list(starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 0]) - g2_psf += list(starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 1]) - size_psf += list(starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 2] ** 2) + g1_psf += list( + starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 0] + ) + g2_psf += list( + starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 1] + ) + size_psf += list( + starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 2] ** 2 + ) g1 += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 0]) g2 += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 1]) - size += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 2] ** 2) + size += list( + starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 2] ** 2 + ) # flags - flag_psf += list(starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 3]) - flag_star += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 3]) + flag_psf += list( + starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 3] + ) + flag_star += list( + starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 3] + ) # ccd id list ccd_nb += list(starcat_j[self._hdu_table].data["CCD_ID_LIST"]) @@ -414,7 +440,9 @@ def process(self): ) # Mask and transform to numpy arrays - flagmask = np.abs(np.array(flag_star) - 1) * np.abs(np.array(flag_psf) - 1) + flagmask = np.abs(np.array(flag_star) - 1) * np.abs( + np.array(flag_psf) - 1 + ) psf_e1 = np.array(g1_psf)[flagmask.astype(bool)] psf_e2 = np.array(g2_psf)[flagmask.astype(bool)] psf_r2 = np.array(size_psf)[flagmask.astype(bool)] @@ -509,7 +537,9 @@ def process(self): mag, snr, psfex_acc = [], [], [] ccd_nb = [] - self._w_log.info(f"Merging {len(self._input_file_list)} star catalogues") + self._w_log.info( + f"Merging {len(self._input_file_list)} star catalogues" + ) for name in self._input_file_list: starcat_j = fits.open(name[0], memmap=False) @@ -694,7 +724,9 @@ def process(self): mag, snr = [], [] ccd_nb = [] - self._w_log.info(f"Merging {len(self._input_file_list)} star catalogues") + self._w_log.info( + f"Merging {len(self._input_file_list)} star catalogues" + ) for name in self._input_file_list: starcat_j = fits.open(name[0], memmap=False) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index e7eab4a9..bbecafe8 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -71,7 +71,8 @@ def __init__( if len(input_file_list) != 6: raise IndexError( - f"Input file list has length {len(input_file_list)}," + " required is 6" + f"Input file list has length {len(input_file_list)}," + + " required is 6" ) self._tile_cat_path = input_file_list[0] @@ -162,7 +163,9 @@ def get_prior(self): F_prior = ngmix.priors.FlatPrior(Fminval, Fmaxval) # Joint prior, combine all individual priors - prior = ngmix.joint_prior.PriorSimpleSep(cen_prior, g_prior, T_prior, F_prior) + prior = ngmix.joint_prior.PriorSimpleSep( + cen_prior, g_prior, T_prior, F_prior + ) return prior @@ -220,7 +223,10 @@ def compile_results(self, results): for idx in range(len(results)): for name in names: - mag = -2.5 * np.log10(results[idx][name]["flux"]) + self._zero_point + mag = ( + -2.5 * np.log10(results[idx][name]["flux"]) + + self._zero_point + ) mag_err = np.abs( -2.5 * results[idx][name]["flux_err"] @@ -228,11 +234,19 @@ def compile_results(self, results): ) output_dict[name]["id"].append(results[idx]["obj_id"]) - output_dict[name]["n_epoch_model"].append(results[idx]["n_epoch_model"]) - output_dict[name]["moments_fail"].append(results[idx]["moments_fail"]) + output_dict[name]["n_epoch_model"].append( + results[idx]["n_epoch_model"] + ) + output_dict[name]["moments_fail"].append( + results[idx]["moments_fail"] + ) output_dict[name]["ntry_fit"].append(results[idx][name]["ntry"]) - output_dict[name]["g1_psfo_ngmix"].append(results[idx]["g_PSFo"][0]) - output_dict[name]["g2_psfo_ngmix"].append(results[idx]["g_PSFo"][1]) + output_dict[name]["g1_psfo_ngmix"].append( + results[idx]["g_PSFo"][0] + ) + output_dict[name]["g2_psfo_ngmix"].append( + results[idx]["g_PSFo"][1] + ) output_dict[name]["g1_err_psfo_ngmix"].append( results[idx]["g_err_PSFo"][0] ) @@ -240,18 +254,30 @@ def compile_results(self, results): results[idx]["g_err_PSFo"][1] ) output_dict[name]["T_psfo_ngmix"].append(results[idx]["T_PSFo"]) - output_dict[name]["T_err_psfo_ngmix"].append(results[idx]["T_err_PSFo"]) + output_dict[name]["T_err_psfo_ngmix"].append( + results[idx]["T_err_PSFo"] + ) output_dict[name]["g1"].append(results[idx][name]["g"][0]) - output_dict[name]["g1_err"].append(results[idx][name]["pars_err"][2]) + output_dict[name]["g1_err"].append( + results[idx][name]["pars_err"][2] + ) output_dict[name]["g2"].append(results[idx][name]["g"][1]) - output_dict[name]["g2_err"].append(results[idx][name]["pars_err"][3]) + output_dict[name]["g2_err"].append( + results[idx][name]["pars_err"][3] + ) output_dict[name]["T"].append(results[idx][name]["T"]) output_dict[name]["T_err"].append(results[idx][name]["T_err"]) output_dict[name]["Tpsf"].append(results[idx][name]["Tpsf"]) - output_dict[name]["g1_psf"].append(results[idx][name]["gpsf"][0]) - output_dict[name]["g2_psf"].append(results[idx][name]["gpsf"][1]) + output_dict[name]["g1_psf"].append( + results[idx][name]["gpsf"][0] + ) + output_dict[name]["g2_psf"].append( + results[idx][name]["gpsf"][1] + ) output_dict[name]["flux"].append(results[idx][name]["flux"]) - output_dict[name]["flux_err"].append(results[idx][name]["flux_err"]) + output_dict[name]["flux_err"].append( + results[idx][name]["flux_err"] + ) output_dict[name]["mag"].append(mag) output_dict[name]["mag_err"].append(mag_err) @@ -263,7 +289,9 @@ def compile_results(self, results): raise KeyError("No SNR key (s2n, s2n_r) found in results") output_dict[name]["flags"].append(results[idx][name]["flags"]) - output_dict[name]["mcal_flags"].append(results[idx]["mcal_flags"]) + output_dict[name]["mcal_flags"].append( + results[idx]["mcal_flags"] + ) return output_dict @@ -350,18 +378,24 @@ def process(self): for expccd_name_tmp in psf_expccd_name: exp_name, ccd_n = re.split("-", expccd_name_tmp) - gal_vign_tmp = gal_vign_cat[str(id_tmp)][expccd_name_tmp]["VIGNET"] + gal_vign_tmp = gal_vign_cat[str(id_tmp)][expccd_name_tmp][ + "VIGNET" + ] if len(np.where(gal_vign_tmp.ravel() == 0)[0]) != 0: continue - bkg_vign_tmp = bkg_vign_cat[str(id_tmp)][expccd_name_tmp]["VIGNET"] + bkg_vign_tmp = bkg_vign_cat[str(id_tmp)][expccd_name_tmp][ + "VIGNET" + ] gal_vign_sub_bkg = gal_vign_tmp - bkg_vign_tmp tile_vign_tmp = Ngmix.MegaCamFlip( np.copy(tile_vign[i_tile]), int(ccd_n) ) - flag_vign_tmp = flag_vign_cat[str(id_tmp)][expccd_name_tmp]["VIGNET"] + flag_vign_tmp = flag_vign_cat[str(id_tmp)][expccd_name_tmp][ + "VIGNET" + ] flag_vign_tmp[np.where(tile_vign_tmp == -1e30)] = 2**10 v_flag_tmp = flag_vign_tmp.ravel() if len(np.where(v_flag_tmp != 0)[0]) / (51 * 51) > 1 / 3.0: @@ -386,7 +420,9 @@ def process(self): weight_vign_scaled = weight_vign_tmp * 1 / Fscale**2 gal_vign.append(gal_vign_scaled) - psf_vign.append(psf_vign_cat[str(id_tmp)][expccd_name_tmp]["VIGNET"]) + psf_vign.append( + psf_vign_cat[str(id_tmp)][expccd_name_tmp]["VIGNET"] + ) sigma_psf.append( psf_vign_cat[str(id_tmp)][expccd_name_tmp]["SHAPES"][ "SIGMA_PSF_HSM" @@ -410,7 +446,9 @@ def process(self): self._pixel_scale, ) except Exception as ee: - self._w_log.info(f"ngmix failed for object ID={id_tmp}.\nMessage: {ee}") + self._w_log.info( + f"ngmix failed for object ID={id_tmp}.\nMessage: {ee}" + ) continue res["obj_id"] = id_tmp @@ -494,7 +532,9 @@ def get_guess( error_msg = hsm_shape.error_message if error_msg != "": - raise galsim.hsm.GalSimHSMError(f"Error in adaptive moments :\n{error_msg}") + raise galsim.hsm.GalSimHSMError( + f"Error in adaptive moments :\n{error_msg}" + ) if guess_flux_unit == "img": guess_flux = hsm_shape.moments_amp @@ -757,7 +797,9 @@ def do_ngmix_metacal( # Gal guess try: - gal_guess_tmp = get_guess(gals[n_e], pixel_scale, guess_size_type="sigma") + gal_guess_tmp = get_guess( + gals[n_e], pixel_scale, guess_size_type="sigma" + ) except Exception: gal_guess_flag = False gal_guess_tmp = np.array([0.0, 0.0, 0.0, 0.0, 1, 100]) @@ -856,7 +898,9 @@ def do_ngmix_metacal( for key in sorted(obs_dict_mcal): - fres = make_galsimfit(obs_dict_mcal[key], gal_model, gal_pars, prior=prior) + fres = make_galsimfit( + obs_dict_mcal[key], gal_model, gal_pars, prior=prior + ) res["mcal_flags"] |= fres["flags"] tres = {} diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 7e781d3f..80d8e2ea 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -12,7 +12,11 @@ @module_runner( version="0.0.1", - input_module=["sextractor_runner", "psfex_interp_runner", "vignetmaker_runner"], + input_module=[ + "sextractor_runner", + "psfex_interp_runner", + "vignetmaker_runner", + ], file_pattern=[ "tile_sexcat", "image", diff --git a/src/shapepipe/modules/pastecat_package/pastecat.py b/src/shapepipe/modules/pastecat_package/pastecat.py index 260ba505..35941018 100644 --- a/src/shapepipe/modules/pastecat_package/pastecat.py +++ b/src/shapepipe/modules/pastecat_package/pastecat.py @@ -70,7 +70,8 @@ def process(self): """ # Create output catalogue pasted_cat = file_io.FITSCatalogue( - self._output_path, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite + self._output_path, + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, ) for idx, input_file in enumerate(self._input_file_list): @@ -93,7 +94,8 @@ def process(self): + "not found in input catalogue" ) if not ( - data[self._check_col_name] == data_prev[self._check_col_name] + data[self._check_col_name] + == data_prev[self._check_col_name] ).all(): raise Exception( f"Column check using key" diff --git a/src/shapepipe/modules/pastecat_runner.py b/src/shapepipe/modules/pastecat_runner.py index 617bcd7a..4f659bff 100644 --- a/src/shapepipe/modules/pastecat_runner.py +++ b/src/shapepipe/modules/pastecat_runner.py @@ -64,7 +64,9 @@ def paste_cat_runner( file_ext = "fits" # Set output path - output_path = f'{run_dirs["output"]}/{prefix}' + f"{file_number_string}.{file_ext}" + output_path = ( + f'{run_dirs["output"]}/{prefix}' + f"{file_number_string}.{file_ext}" + ) # Create rand cat class instance paste_cat_inst = PasteCat( diff --git a/src/shapepipe/modules/psfex_interp_package/psfex_interp.py b/src/shapepipe/modules/psfex_interp_package/psfex_interp.py index e0c26750..d98cc337 100644 --- a/src/shapepipe/modules/psfex_interp_package/psfex_interp.py +++ b/src/shapepipe/modules/psfex_interp_package/psfex_interp.py @@ -72,8 +72,14 @@ def interpsfex(dotpsfpath, pos, thresh_star, thresh_chi2): return PSF_basis[0, :, :] # scale coordinates - x_interp, x_scale = (PSF_model.header["POLZERO1"], PSF_model.header["POLSCAL1"]) - y_interp, y_scale = (PSF_model.header["POLZERO2"], PSF_model.header["POLSCAL2"]) + x_interp, x_scale = ( + PSF_model.header["POLZERO1"], + PSF_model.header["POLSCAL1"], + ) + y_interp, y_scale = ( + PSF_model.header["POLZERO2"], + PSF_model.header["POLSCAL2"], + ) xs, ys = (pos[:, 0] - x_interp) / x_scale, (pos[:, 1] - y_interp) / y_scale # compute polynomial coefficients @@ -82,7 +88,10 @@ def interpsfex(dotpsfpath, pos, thresh_star, thresh_chi2): [ np.concatenate( [ - [(x**idx_j) * (y**idx_i) for idx_j in range(deg - idx_i + 1)] + [ + (x**idx_j) * (y**idx_i) + for idx_j in range(deg - idx_i + 1) + ] for idx_i in range(1, deg + 1) ] ) @@ -201,7 +210,10 @@ def process(self): if self.interp_PSFs is None: self._interpolate() - if isinstance(self.interp_PSFs, str) and self.interp_PSFs == NOT_ENOUGH_STARS: + if ( + isinstance(self.interp_PSFs, str) + and self.interp_PSFs == NOT_ENOUGH_STARS + ): self._w_log.info( "Not enough stars to interpolate the psf in the file " + f"{self._dotpsf_path}." @@ -210,7 +222,10 @@ def process(self): self._w_log.info( f"Bad chi2 for the psf model in the file {self._dotpsf_path}." ) - elif isinstance(self.interp_PSFs, str) and self.interp_PSFs == FILE_NOT_FOUND: + elif ( + isinstance(self.interp_PSFs, str) + and self.interp_PSFs == FILE_NOT_FOUND + ): self._w_log.info(f"Psf model file {self._dotpsf_path} not found.") else: if self._compute_shape: @@ -294,7 +309,8 @@ def _get_psfshapes(self): raise ImportError("Galsim is required to get shapes information") psf_moms = [ - hsm.FindAdaptiveMom(Image(psf), strict=False) for psf in self.interp_PSFs + hsm.FindAdaptiveMom(Image(psf), strict=False) + for psf in self.interp_PSFs ] self.psf_shapes = np.array( @@ -353,7 +369,10 @@ def process_validation(self, psfex_cat_path): if self.interp_PSFs is None: self._interpolate() - if isinstance(self.interp_PSFs, str) and self.interp_PSFs == NOT_ENOUGH_STARS: + if ( + isinstance(self.interp_PSFs, str) + and self.interp_PSFs == NOT_ENOUGH_STARS + ): self._w_log.info( "Not enough stars to interpolate the psf in the file " + f"{self._dotpsf_path}." @@ -362,7 +381,10 @@ def process_validation(self, psfex_cat_path): self._w_log.info( f"Bad chi2 for the psf model in the file {self._dotpsf_path}." ) - elif isinstance(self.interp_PSFs, str) and self.interp_PSFs == FILE_NOT_FOUND: + elif ( + isinstance(self.interp_PSFs, str) + and self.interp_PSFs == FILE_NOT_FOUND + ): self._w_log.info(f"Psf model file {self._dotpsf_path} not found.") else: star_cat = file_io.FITSCatalogue( @@ -441,9 +463,15 @@ def _get_psfexcatdict(self, psfex_cat_path): psfex_cat.open() psfex_cat_dict = {} - psfex_cat_dict["SOURCE_NUMBER"] = np.copy(psfex_cat.get_data()["SOURCE_NUMBER"]) - psfex_cat_dict["DELTAX_IMAGE"] = np.copy(psfex_cat.get_data()["DELTAX_IMAGE"]) - psfex_cat_dict["DELTAY_IMAGE"] = np.copy(psfex_cat.get_data()["DELTAY_IMAGE"]) + psfex_cat_dict["SOURCE_NUMBER"] = np.copy( + psfex_cat.get_data()["SOURCE_NUMBER"] + ) + psfex_cat_dict["DELTAX_IMAGE"] = np.copy( + psfex_cat.get_data()["DELTAX_IMAGE"] + ) + psfex_cat_dict["DELTAY_IMAGE"] = np.copy( + psfex_cat.get_data()["DELTAY_IMAGE"] + ) psfex_cat_dict["CHI2_PSF"] = np.copy(psfex_cat.get_data()["CHI2_PSF"]) return psfex_cat_dict @@ -554,7 +582,9 @@ def _interpolate_me(self): list_ext_name = cat.get_ext_name() hdu_ind = [ - idx for idx in range(len(list_ext_name)) if "EPOCH" in list_ext_name[idx] + idx + for idx in range(len(list_ext_name)) + if "EPOCH" in list_ext_name[idx] ] final_list = [] @@ -599,7 +629,10 @@ def _interpolate_me(self): + "lose an epoch." ) continue - if isinstance(self.interp_PSFs, str) and self.interp_PSFs == BAD_CHI2: + if ( + isinstance(self.interp_PSFs, str) + and self.interp_PSFs == BAD_CHI2 + ): self._w_log.info( f"Bad chi2 for the psf model in the ccd {ccd} of the " + f"exposure {exp_name}. Object inside this ccd will " @@ -619,7 +652,9 @@ def _interpolate_me(self): if array_psf is None: array_psf = np.copy(self.interp_PSFs) else: - array_psf = np.concatenate((array_psf, np.copy(self.interp_PSFs))) + array_psf = np.concatenate( + (array_psf, np.copy(self.interp_PSFs)) + ) if array_id is None: array_id = np.copy(obj_id) @@ -646,9 +681,13 @@ def _interpolate_me(self): if array_exp_name is None: array_exp_name = exp_name_tmp else: - array_exp_name = np.concatenate((array_exp_name, exp_name_tmp)) + array_exp_name = np.concatenate( + (array_exp_name, exp_name_tmp) + ) - final_list.append([array_id, array_psf, array_shape, array_exp_name]) + final_list.append( + [array_id, array_psf, array_shape, array_exp_name] + ) self._f_wcs_file.close() cat.close() @@ -662,15 +701,23 @@ def _interpolate_me(self): where_res = np.where(final_list[j][0] == id_tmp)[0] if len(where_res) != 0: output_dict[id_tmp][final_list[j][3][where_res[0]]] = {} - output_dict[id_tmp][final_list[j][3][where_res[0]]]["VIGNET"] = ( - final_list[j][1][where_res[0]] - ) + output_dict[id_tmp][final_list[j][3][where_res[0]]][ + "VIGNET" + ] = final_list[j][1][where_res[0]] if self._compute_shape: shape_dict = {} - shape_dict["E1_PSF_HSM"] = final_list[j][2][where_res[0]][0] - shape_dict["E2_PSF_HSM"] = final_list[j][2][where_res[0]][1] - shape_dict["SIGMA_PSF_HSM"] = final_list[j][2][where_res[0]][2] - shape_dict["FLAG_PSF_HSM"] = final_list[j][2][where_res[0]][3] + shape_dict["E1_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][0] + shape_dict["E2_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][1] + shape_dict["SIGMA_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][2] + shape_dict["FLAG_PSF_HSM"] = final_list[j][2][ + where_res[0] + ][3] output_dict[id_tmp][final_list[j][3][where_res[0]]][ "SHAPES" ] = shape_dict @@ -694,7 +741,9 @@ def _write_output_me(self, output_dict): Dictionnary of outputs to save """ - output_file = SqliteDict(self._output_path + self._img_number + ".sqlite") + output_file = SqliteDict( + self._output_path + self._img_number + ".sqlite" + ) for idx in output_dict.keys(): output_file[str(idx)] = output_dict[idx] output_file.commit() diff --git a/src/shapepipe/modules/psfex_runner.py b/src/shapepipe/modules/psfex_runner.py index e636e8f7..d9c515e5 100644 --- a/src/shapepipe/modules/psfex_runner.py +++ b/src/shapepipe/modules/psfex_runner.py @@ -29,7 +29,9 @@ def psfex_runner( """Define The PSFEx Runner.""" # Extract psfex run configurations if config.has_option(module_config_sec, "EXEC_PATH"): - psfex_executable_path = config.getexpanded(module_config_sec, "EXEC_PATH") + psfex_executable_path = config.getexpanded( + module_config_sec, "EXEC_PATH" + ) else: psfex_executable_path = "psfex" output_dir = run_dirs["output"] diff --git a/src/shapepipe/modules/python_example_runner.py b/src/shapepipe/modules/python_example_runner.py index 587558e3..f5016f08 100644 --- a/src/shapepipe/modules/python_example_runner.py +++ b/src/shapepipe/modules/python_example_runner.py @@ -41,7 +41,9 @@ def python_example_runner( ): """Define The Python Example Runner.""" # Set output file name - output_file_name = f'{run_dirs["output"]}/pyex_output{file_number_string}.cat' + output_file_name = ( + f'{run_dirs["output"]}/pyex_output{file_number_string}.cat' + ) # Retrieve log message from config file message = config.get(module_config_sec, "MESSAGE") diff --git a/src/shapepipe/modules/random_cat_package/random_cat.py b/src/shapepipe/modules/random_cat_package/random_cat.py index f8456155..feb69ff9 100644 --- a/src/shapepipe/modules/random_cat_package/random_cat.py +++ b/src/shapepipe/modules/random_cat_package/random_cat.py @@ -167,7 +167,9 @@ def process(self): n_obj = self._n_rand else: # Compute number of objects from density - n_obj = int(self._n_rand / area_deg2 * area_deg2_eff / area_deg2) + n_obj = int( + self._n_rand / area_deg2 * area_deg2_eff / area_deg2 + ) # Check that a reasonably large number of pixels is not masked if n_unmasked < n_obj: @@ -233,4 +235,6 @@ def process(self): # Write area information to log file self._w_log.info(f"Total area = {area_deg2:.4f} deg^2") self._w_log.info(f"Unmasked area = {area_deg2_eff:.4f} deg^2") - self._w_log.info(f"Ratio masked to total pixels = {n_unmasked / n_pix:.3f}") + self._w_log.info( + f"Ratio masked to total pixels = {n_unmasked / n_pix:.3f}" + ) diff --git a/src/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py index f53b54c5..665792fe 100644 --- a/src/shapepipe/modules/random_cat_runner.py +++ b/src/shapepipe/modules/random_cat_runner.py @@ -32,7 +32,9 @@ def random_cat_runner( # Set output file name if config.has_option(module_config_sec, "OUTPUT_FILE_PATTERN"): - output_file_pattern = config.get(module_config_sec, "OUTPUT_FILE_PATTERN") + output_file_pattern = config.get( + module_config_sec, "OUTPUT_FILE_PATTERN" + ) else: output_file_pattern = "random_cat" @@ -47,12 +49,16 @@ def random_cat_runner( density = False # Get healpix output options - save_mask_as_healpix = config.getboolean(module_config_sec, "SAVE_MASK_AS_HEALPIX") + save_mask_as_healpix = config.getboolean( + module_config_sec, "SAVE_MASK_AS_HEALPIX" + ) if save_mask_as_healpix: healpix_options = {} for option_trunc in ["FILE_BASE", "NSIDE"]: option = f"HEALPIX_OUT_{option_trunc}" - healpix_options[option_trunc] = config.get(module_config_sec, option) + healpix_options[option_trunc] = config.get( + module_config_sec, option + ) # Create rand cat class instance rand_cat_inst = RandomCat( input_image_name, diff --git a/src/shapepipe/modules/serial_example_runner.py b/src/shapepipe/modules/serial_example_runner.py index c5028472..61cfc3ed 100644 --- a/src/shapepipe/modules/serial_example_runner.py +++ b/src/shapepipe/modules/serial_example_runner.py @@ -55,7 +55,9 @@ def serial_example_runner( w_log, ): """Define The Serial Example Runner.""" - output_file_name = f'{run_dirs["output"]}/serial_outputfile_number_string.cat' + output_file_name = ( + f'{run_dirs["output"]}/serial_outputfile_number_string.cat' + ) inst = Dummy() inst.read_files(input_file_list) diff --git a/src/shapepipe/modules/setools_package/setools.py b/src/shapepipe/modules/setools_package/setools.py index 4f7925da..ec0606e9 100644 --- a/src/shapepipe/modules/setools_package/setools.py +++ b/src/shapepipe/modules/setools_package/setools.py @@ -231,7 +231,9 @@ def read(self): try: rand_split_name = re.split(":", sec)[1] except Exception: - rand_split_name = f"rand_split_{len(self._rand_split) + 1}" + rand_split_name = ( + f"rand_split_{len(self._rand_split) + 1}" + ) self._rand_split[rand_split_name] = [] else: raise ValueError( @@ -567,7 +569,8 @@ def _make_plot(self): self.plot[key][ss[0]][ss[1]] = s[1] else: raise ValueError( - "Plot keyword not in correct format (key or key_i)" + f": {idx}" + "Plot keyword not in correct format (key or key_i)" + + f": {idx}" ) def _make_new_cat(self): @@ -629,7 +632,9 @@ def _make_rand_split(self): for idx in self._rand_split[key]: s = re.split("=", idx) if len(s) != 2: - raise ValueError(f"Not a valid format : {self._rand_split[key][0]}") + raise ValueError( + f"Not a valid format : {self._rand_split[key][0]}" + ) if s[0] == "RATIO": try: ratio = float(s[1]) @@ -745,7 +750,12 @@ def __init__(self, plot_dict, catalogue, output_path, mask_dict=None): elif self._plot["TYPE"]["0"] in ["scatter", "SCATTER"]: self._check_key_for_plot(["X", "Y"]) self._make_scatter() - elif self._plot["TYPE"]["0"] in ["histogram", "hist", "HISTOGRAM", "HIST"]: + elif self._plot["TYPE"]["0"] in [ + "histogram", + "hist", + "HISTOGRAM", + "HIST", + ]: self._check_key_for_plot(["Y"]) self._make_hist() else: @@ -1160,7 +1170,9 @@ def _make_hist(self): bins = int(self._plot["BIN"][key]) except Exception: if len(self._plot["BIN"]) == 1: - bins = int(self._plot["BIN"][self._plot["BIN"].keys()[0]]) + bins = int( + self._plot["BIN"][self._plot["BIN"].keys()[0]] + ) else: bins = 50 if "ALPHA" in self._plot.keys(): diff --git a/src/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py index d57c80a8..343e8b9d 100644 --- a/src/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/src/shapepipe/modules/sextractor_package/sextractor_script.py @@ -40,7 +40,9 @@ def get_header_value(image_path, key): try: val = float(val) except Exception: - raise ValueError(f"The key {key} does not return a float value. Got {val}") + raise ValueError( + f"The key {key} does not return a float value. Got {val}" + ) return val @@ -118,7 +120,10 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): n_epoch[ind] += 1 exp_name = np.array([exp_list[idx] for n in range(len(obj_id))]) a = np.array( - [(obj_id[ii], exp_name[ii], pos_tmp[ii]) for ii in range(len(exp_name))], + [ + (obj_id[ii], exp_name[ii], pos_tmp[ii]) + for ii in range(len(exp_name)) + ], dtype=[ ("NUMBER", obj_id.dtype), ("EXP_NAME", exp_name.dtype), @@ -216,7 +221,11 @@ def __init__( self._path_dot_conv = path_dot_conv self.set_input_files( - use_weight, use_flag, use_psf, use_detection_image, use_detection_weight + use_weight, + use_flag, + use_psf, + use_detection_image, + use_detection_weight, ) # Collect optional arguments for SExtractor @@ -291,11 +300,15 @@ def set_input_files( extra += 1 if use_flag: - self._cmd_line_extra += " -FLAG_IMAGE " + f"{self._all_input_path[extra]}" + self._cmd_line_extra += ( + " -FLAG_IMAGE " + f"{self._all_input_path[extra]}" + ) extra += 1 if use_psf: - self._cmd_line_extra += " -PSF_NAME " + f"{self._all_input_path[extra]}" + self._cmd_line_extra += ( + " -PSF_NAME " + f"{self._all_input_path[extra]}" + ) extra += 1 # Check for separate files for detection and measurement @@ -303,7 +316,8 @@ def set_input_files( # First, consistency checks if use_detect_weight and not use_detect_img: raise ValueError( - "DETECTION_WEIGHT cannot be True " + "if DETECTION_IMAGE is False" + "DETECTION_WEIGHT cannot be True " + + "if DETECTION_IMAGE is False" ) if use_detect_weight and not use_weight: raise ValueError( @@ -373,7 +387,9 @@ def get_background(self, use_bkg, bkg_key=None): """ if use_bkg and not isinstance(bkg_key, type(None)): bkg_value = get_header_value(self._meas_img_path, bkg_key) - self._cmd_line_extra += f" -BACK_TYPE MANUAL -BACK_VALUE {bkg_value}" + self._cmd_line_extra += ( + f" -BACK_TYPE MANUAL -BACK_VALUE {bkg_value}" + ) def get_check_image(self, check_image): """Get Check Image. diff --git a/src/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py index 0f8e4886..283968e4 100644 --- a/src/shapepipe/modules/sextractor_runner.py +++ b/src/shapepipe/modules/sextractor_runner.py @@ -104,7 +104,9 @@ def sextractor_runner( f_wcs_path = config.getexpanded(module_config_sec, "LOG_WCS") pos_params = config.getlist(module_config_sec, "WORLD_POSITION") ccd_size = config.getlist(module_config_sec, "CCD_SIZE") - ss.make_post_process(ss_inst.path_output_file, f_wcs_path, pos_params, ccd_size) + ss.make_post_process( + ss_inst.path_output_file, f_wcs_path, pos_params, ccd_size + ) # Return stdout and stderr return stdout, stderr diff --git a/src/shapepipe/modules/split_exp_package/split_exp.py b/src/shapepipe/modules/split_exp_package/split_exp.py index c8049f8d..ced6cefb 100644 --- a/src/shapepipe/modules/split_exp_package/split_exp.py +++ b/src/shapepipe/modules/split_exp_package/split_exp.py @@ -37,7 +37,12 @@ class SplitExposures(object): """ def __init__( - self, input_file_list, output_dir, file_number_string, output_suffix, n_hdu + self, + input_file_list, + output_dir, + file_number_string, + output_suffix, + n_hdu, ): self._input_file_list = input_file_list @@ -52,7 +57,9 @@ def process(self): Process the splitting of single-exposure images. """ - for exp_path, output_suffix in zip(self._input_file_list, self._output_suffix): + for exp_path, output_suffix in zip( + self._input_file_list, self._output_suffix + ): transf_int = "flag" in output_suffix transf_coord = "image" in output_suffix @@ -114,5 +121,7 @@ def create_hdus( header_file[idx - 1] = {"WCS": w, "header": h.tostring()} if save_header: - file_name = f"{self._output_dir}/headers{self._file_number_string}.npy" + file_name = ( + f"{self._output_dir}/headers{self._file_number_string}.npy" + ) np.save(file_name, header_file) diff --git a/src/shapepipe/modules/split_exp_runner.py b/src/shapepipe/modules/split_exp_runner.py index 304faac9..77e6010d 100644 --- a/src/shapepipe/modules/split_exp_runner.py +++ b/src/shapepipe/modules/split_exp_runner.py @@ -34,7 +34,11 @@ def split_exp_runner( # Create split exposures class instance split_inst = SplitExposures( - input_file_list, run_dirs["output"], file_number_string, output_suffix, n_hdu + input_file_list, + run_dirs["output"], + file_number_string, + output_suffix, + n_hdu, ) # Process splitting diff --git a/src/shapepipe/modules/spread_model_package/spread_model.py b/src/shapepipe/modules/spread_model_package/spread_model.py index 419badba..659b0bb8 100644 --- a/src/shapepipe/modules/spread_model_package/spread_model.py +++ b/src/shapepipe/modules/spread_model_package/spread_model.py @@ -220,7 +220,10 @@ def process(self): obj_sigma_tmp = np.mean(sigma_list) obj_weight_tmp = weigh_vign[idx] obj_model_tmp, obj_psf_tmp = get_model( - obj_sigma_tmp, obj_flux_tmp, obj_vign_tmp.shape, self._pixel_scale + obj_sigma_tmp, + obj_flux_tmp, + obj_vign_tmp.shape, + self._pixel_scale, ) obj_sm, obj_sm_err = get_sm( @@ -238,7 +241,9 @@ def process(self): psf_cat.close() - self.save_results(spread_model_final, spread_model_err_final, obj_mag, obj_id) + self.save_results( + spread_model_final, spread_model_err_final, obj_mag, obj_id + ) def save_results(self, sm, sm_err, mag, number): """Save Results. @@ -274,7 +279,9 @@ def save_results(self, sm, sm_err, mag, number): "SPREAD_MODEL": sm, "SPREADERR_MODEL": sm_err, } - new_cat.save_as_fits(data=dict_data, sex_cat_path=self._sex_cat_path) + new_cat.save_as_fits( + data=dict_data, sex_cat_path=self._sex_cat_path + ) elif self._output_mode == "add": ori_cat = file_io.FITSCatalogue( self._sex_cat_path, @@ -286,7 +293,9 @@ def save_results(self, sm, sm_err, mag, number): SEx_catalogue=True, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, ) - ori_cat.add_col("SPREAD_MODEL", sm, new_cat=True, new_cat_inst=new_cat) + ori_cat.add_col( + "SPREAD_MODEL", sm, new_cat=True, new_cat_inst=new_cat + ) ori_cat.close() new_cat.open() new_cat.add_col("SPREADERR_MODEL", sm_err) diff --git a/src/shapepipe/modules/spread_model_runner.py b/src/shapepipe/modules/spread_model_runner.py index 2702c258..16c9a2d3 100644 --- a/src/shapepipe/modules/spread_model_runner.py +++ b/src/shapepipe/modules/spread_model_runner.py @@ -12,7 +12,11 @@ @module_runner( version="1.1", - input_module=["sextractor_runner", "psfex_interp_runner", "vignetmaker_runner"], + input_module=[ + "sextractor_runner", + "psfex_interp_runner", + "vignetmaker_runner", + ], file_pattern=["sexcat", "galaxy_psf", "weight_vign"], file_ext=[".fits", ".sqlite", ".fits"], depends=["numpy", "galsim"], diff --git a/src/shapepipe/modules/vignetmaker_package/vignetmaker.py b/src/shapepipe/modules/vignetmaker_package/vignetmaker.py index 01e5cc75..f865506c 100644 --- a/src/shapepipe/modules/vignetmaker_package/vignetmaker.py +++ b/src/shapepipe/modules/vignetmaker_package/vignetmaker.py @@ -77,7 +77,8 @@ def process(self, image_path_list, rad, prefix): pos = convert_pos(image_path) else: raise ValueError( - "Coordinates type must be in : PIX (pixel), " + "SPHE (spherical)." + "Coordinates type must be in : PIX (pixel), " + + "SPHE (spherical)." ) vign = self._get_stamp(image_path, pos - 1, rad) @@ -207,7 +208,9 @@ def _get_stamp_me(self, image_dir, image_pattern): n_epoch = np.copy(cat.get_data()["N_EPOCH"]) list_ext_name = cat.get_ext_name() - hdu_ind = [i for i in range(len(list_ext_name)) if "EPOCH" in list_ext_name[i]] + hdu_ind = [ + i for i in range(len(list_ext_name)) if "EPOCH" in list_ext_name[i] + ] final_list = [] for hdu_index in hdu_ind: @@ -263,7 +266,9 @@ def _get_stamp_me(self, image_dir, image_pattern): if array_exp_name is None: array_exp_name = exp_name_tmp else: - array_exp_name = np.concatenate((array_exp_name, exp_name_tmp)) + array_exp_name = np.concatenate( + (array_exp_name, exp_name_tmp) + ) final_list.append([array_id, array_vign, array_exp_name]) diff --git a/src/shapepipe/pipeline/args.py b/src/shapepipe/pipeline/args.py index 1012e0cd..d5c9d645 100644 --- a/src/shapepipe/pipeline/args.py +++ b/src/shapepipe/pipeline/args.py @@ -122,7 +122,9 @@ def create_arg_parser(): optional.add_argument( "-l", "--list_modules", - action=print_message(f"ShapePipe modules currently available:\n{module_str()}"), + action=print_message( + f"ShapePipe modules currently available:\n{module_str()}" + ), help="list modules currently available and exit", ) diff --git a/src/shapepipe/pipeline/config.py b/src/shapepipe/pipeline/config.py index 309f5286..f43bf734 100644 --- a/src/shapepipe/pipeline/config.py +++ b/src/shapepipe/pipeline/config.py @@ -59,7 +59,8 @@ def getlist(self, section, option, delimiter=","): """ return [ - opt.strip() for opt in self.getexpanded(section, option).split(delimiter) + opt.strip() + for opt in self.getexpanded(section, option).split(delimiter) ] diff --git a/src/shapepipe/pipeline/dependency_handler.py b/src/shapepipe/pipeline/dependency_handler.py index 94546a03..f4779e5e 100644 --- a/src/shapepipe/pipeline/dependency_handler.py +++ b/src/shapepipe/pipeline/dependency_handler.py @@ -159,7 +159,9 @@ def _get_indices(array, value): List of indices """ - return [index for index, element in enumerate(array) if element == value] + return [ + index for index, element in enumerate(array) if element == value + ] @classmethod def _slice_col_val(cls, array, col, value): @@ -271,7 +273,10 @@ def _split_strings(self): """ self._depend_arr = list( - map(list, zip(*[self._split_string(string) for string in self.depend])) + map( + list, + zip(*[self._split_string(string) for string in self.depend]), + ) ) self._dependency_set = set(self._depend_arr[0]) @@ -292,7 +297,9 @@ def _unique_dependencies(self): subset = self._slice_col_val( subset, 2, - str(max([self._convert_to_float(ver) for ver in subset[2]])), + str( + max([self._convert_to_float(ver) for ver in subset[2]]) + ), ) subset = [element[0] for element in self._slice_2d(subset, [0])] @@ -317,7 +324,9 @@ def check_dependencies(self): try: package = importlib.import_module(dependency) except Exception: - raise ImportError(f"Could not import pipeline dependency {dependency}") + raise ImportError( + f"Could not import pipeline dependency {dependency}" + ) if hasattr(package, "__version__"): version = package.__version__ @@ -331,7 +340,9 @@ def check_dependencies(self): else: path = "N/A" - dependency_status_list.append(f" - {package.__name__} {version} {path}") + dependency_status_list.append( + f" - {package.__name__} {version} {path}" + ) return dependency_status_list diff --git a/src/shapepipe/pipeline/execute.py b/src/shapepipe/pipeline/execute.py index 5c97a12e..0a69fae7 100644 --- a/src/shapepipe/pipeline/execute.py +++ b/src/shapepipe/pipeline/execute.py @@ -82,5 +82,6 @@ def is_exe(fpath): if not res: raise OSError( - f"{exe_name} does not appear to be a valid executable on this " + "system." + f"{exe_name} does not appear to be a valid executable on this " + + "system." ) diff --git a/src/shapepipe/pipeline/file_handler.py b/src/shapepipe/pipeline/file_handler.py index 581153dd..8a6263d0 100644 --- a/src/shapepipe/pipeline/file_handler.py +++ b/src/shapepipe/pipeline/file_handler.py @@ -336,7 +336,8 @@ def _check_input_dir_list(self, dir_list): input_dir.extend( [ self.setpath( - self.setpath(run.split(" ")[0], module_run), "output" + self.setpath(run.split(" ")[0], module_run), + "output", ) for run in all_runs ] @@ -457,7 +458,9 @@ def get_module_run_prop(self, module, property): elif property in self._module_dict[module][run_name].keys(): value = self._module_dict[module][run_name][property] else: - raise ValueError(f"Property {property} not found for module {module}.") + raise ValueError( + f"Property {property} not found for module {module}." + ) return value @@ -562,7 +565,9 @@ def _set_module_property(self, module, run_name, property, get_type): prop_val = getattr(self.module_runners[module], property) else: - raise ValueError(f"No value for {property} in {module} could be found.") + raise ValueError( + f"No value for {property} in {module} could be found." + ) # Look for additional module properties for list objects if isinstance(prop_val, list) and self.get_add_module_property( @@ -707,7 +712,9 @@ def _set_module_input_dir(self, module, run_name): dir_set_by = f"{module}.py" # If so, loop through all the input modules defined for the current # module - for input_module in self._module_dict[module][run_name]["input_module"]: + for input_module in self._module_dict[module][run_name][ + "input_module" + ]: # Get the input module name and run input_module, in_mod_run = shared.split_module_run(input_module) # Check if the input module was part of the current pipeline @@ -719,7 +726,9 @@ def _set_module_input_dir(self, module, run_name): # Add the output directory of the input module to # the list of input directories for the current module input_dir.append( - self._module_dict[input_module][in_mod_run]["output_dir"] + self._module_dict[input_module][in_mod_run][ + "output_dir" + ] ) else: # If not, add the last run of the input module to the @@ -744,7 +753,9 @@ def _set_module_input_dir(self, module, run_name): # Add the input directories for the current module to the module # dictionary - self._module_dict[module][run_name]["input_dir"] = self.check_dirs(input_dir) + self._module_dict[module][run_name]["input_dir"] = self.check_dirs( + input_dir + ) self._module_dict[module][run_name]["dir_set_by"] = dir_set_by # Log the input directories and how they were set @@ -787,11 +798,12 @@ def _generate_re_pattern(match_pattern): split_pattern = "|".join(chars).replace(".", r"\.") chars = [f"\\{char}" for char in chars] + [""] num_length = [ - f"\\d{{{len(digits)}}}" for digits in re.split(split_pattern, match_pattern) + f"\\d{{{len(digits)}}}" + for digits in re.split(split_pattern, match_pattern) ] - re_pattern = r"".join([a for b in zip(num_length, chars) for a in b]).replace( - "{1}", "+" - ) + re_pattern = r"".join( + [a for b in zip(num_length, chars) for a in b] + ).replace("{1}", "+") return re.compile(re_pattern) @@ -815,7 +827,9 @@ def _strip_dir_from_file(file_name, dir_list): """ return [ - file_name.replace(_dir + "/", "") for _dir in dir_list if _dir in file_name + file_name.replace(_dir + "/", "") + for _dir in dir_list + if _dir in file_name ][0] @classmethod @@ -856,7 +870,9 @@ def _get_re(cls, num_scheme): return re_pattern - def _save_num_patterns(self, dir_list, re_pattern, pattern, ext, output_file): + def _save_num_patterns( + self, dir_list, re_pattern, pattern, ext, output_file + ): """Save Number Patterns. Save file number patterns to numpy binary, update file patterns and @@ -930,7 +946,8 @@ def _save_num_patterns(self, dir_list, re_pattern, pattern, ext, output_file): if new_pattern != pattern: print( - f'Updating pattern from "{pattern}" to ' + f'"{new_pattern}".' + f'Updating pattern from "{pattern}" to ' + + f'"{new_pattern}".' ) print() @@ -978,7 +995,9 @@ def _save_match_patterns(output_file, mmap_list): np.save( output_file, - reduce(partial(np.intersect1d, assume_unique=True), num_pattern_list), + reduce( + partial(np.intersect1d, assume_unique=True), num_pattern_list + ), ) del num_pattern_list @@ -1124,7 +1143,9 @@ def _save_process_list( temp = [ self._save_num_patterns(dir_list, re_pattern, pattern, ext, np_mmap) - for pattern, ext, np_mmap in zip(pattern_list, ext_list, np_mmap_list) + for pattern, ext, np_mmap in zip( + pattern_list, ext_list, np_mmap_list + ) ] self._save_match_patterns(match_mmap, np_mmap_list) diff --git a/src/shapepipe/pipeline/file_io.py b/src/shapepipe/pipeline/file_io.py index 51a4271e..03d6b869 100644 --- a/src/shapepipe/pipeline/file_io.py +++ b/src/shapepipe/pipeline/file_io.py @@ -274,7 +274,8 @@ def __init__(self, filepath): def __str__(self): return ( - f"File IO *** ERROR ***: catalogue: {self._filepath} " + "is not open" + f"File IO *** ERROR ***: catalogue: {self._filepath} " + + "is not open" ) class DataNotFound(Exception): @@ -1304,9 +1305,13 @@ def get_col_comments(self, hdu_no=None): if hdu_no is None: hdu_no = self.hdu_no hdr_col_types = [ - tt for tt in self._cat_data[hdu_no].header.keys() if "TTYPE" in tt + tt + for tt in self._cat_data[hdu_no].header.keys() + if "TTYPE" in tt + ] + return [ + self._cat_data[hdu_no].header.comments[c] for c in hdr_col_types ] - return [self._cat_data[hdu_no].header.comments[c] for c in hdr_col_types] else: raise BaseCatalogue.catalogueNotOpen(self.fullpath) @@ -1687,7 +1692,9 @@ def _save_to_fits( ) ) - self._cat_data.append(fits.BinTableHDU.from_columns(col_list, name=ext_name)) + self._cat_data.append( + fits.BinTableHDU.from_columns(col_list, name=ext_name) + ) self.close() def _save_from_recarray( diff --git a/src/shapepipe/pipeline/job_handler.py b/src/shapepipe/pipeline/job_handler.py index 9fbdb8cf..15807fb3 100644 --- a/src/shapepipe/pipeline/job_handler.py +++ b/src/shapepipe/pipeline/job_handler.py @@ -104,7 +104,9 @@ def config(self): def config(self, value): if not isinstance(value, ConfigParser): - raise TypeError("config must be an instane of configparser.ConfigParser") + raise TypeError( + "config must be an instane of configparser.ConfigParser" + ) self._config = value @@ -255,7 +257,9 @@ def timeout(self): @timeout.setter def timeout(self, value): - if isinstance(value, type(None)) and self.config.has_option("JOB", "TIMEOUT"): + if isinstance(value, type(None)) and self.config.has_option( + "JOB", "TIMEOUT" + ): value = self.config.get("JOB", "TIMEOUT") value = self.hms2sec(value) if ":" in value else int(value) @@ -334,7 +338,9 @@ def _log_job_parameters(self): batch_info = f" -- Batch size: {self.batch_size}" time_info = f" -- Timeout Limit: {self.timeout}s" - show_batch_into = self.job_type == "parallel" and self.parallel_mode == "smp" + show_batch_into = ( + self.job_type == "parallel" and self.parallel_mode == "smp" + ) self.log.info(text) self.log.info(module_info) @@ -444,7 +450,9 @@ def _check_stderr_status(self): """ for worker_dict in self.worker_dicts: if worker_dict["stderr"]: - self.log.info(f'ERROR: stderr recorded in: {worker_dict["log"]}') + self.log.info( + f'ERROR: stderr recorded in: {worker_dict["log"]}' + ) self.error_count += 1 def _check_missed_processes(self): diff --git a/src/shapepipe/pipeline/run_log.py b/src/shapepipe/pipeline/run_log.py index cb76b89a..a5eb4ebe 100644 --- a/src/shapepipe/pipeline/run_log.py +++ b/src/shapepipe/pipeline/run_log.py @@ -76,7 +76,8 @@ def get_run(self, search_string): elif len(runs) > 1: raise RuntimeError( - "More than one run found matching search string " + f"'{search_string}'" + "More than one run found matching search string " + + f"'{search_string}'" ) return runs[0].split(" ")[0] diff --git a/src/shapepipe/pipeline/shared.py b/src/shapepipe/pipeline/shared.py index 01e8fe24..24984b34 100644 --- a/src/shapepipe/pipeline/shared.py +++ b/src/shapepipe/pipeline/shared.py @@ -108,7 +108,9 @@ def split_module_run(module_str): """ if not isinstance(module_str, str): - raise TypeError(f"Input module_str must be a string not {type(module_str)}.") + raise TypeError( + f"Input module_str must be a string not {type(module_str)}." + ) run_split = "_run_" module_run = module_str diff --git a/src/shapepipe/pipeline/str_handler.py b/src/shapepipe/pipeline/str_handler.py index fb6ba73b..e443e31f 100644 --- a/src/shapepipe/pipeline/str_handler.py +++ b/src/shapepipe/pipeline/str_handler.py @@ -131,7 +131,9 @@ def _compare(self, string): comp = "<|>|<=|>=|==|!=" if len(re.split(comp, string)) != 2: - raise Exception("Only one comparison in [<, >, <=, >=, ==, !=] per line") + raise Exception( + "Only one comparison in [<, >, <=, >=, ==, !=] per line" + ) for operator in ["<=", ">=", "<", ">", "==", "!="]: terms = re.split(operator, string) @@ -361,7 +363,9 @@ def _test_homogeneity(self, *args): param = [args[0], args[1]] n_cells = args[2] else: - raise ValueError("Inputs should be param_1, param_2 [optional], n_cells") + raise ValueError( + "Inputs should be param_1, param_2 [optional], n_cells" + ) if n_param == 2: if len(param[0]) != len(param[1]): @@ -379,10 +383,19 @@ def _test_homogeneity(self, *args): param_min = [] param_max = [] for idx in param: - step = (np.max(idx) - np.min(idx)) / pow(n_cells, 1.0 / float(n_param)) - param_min.append([val for val in np.arange(np.min(idx), np.max(idx), step)]) + step = (np.max(idx) - np.min(idx)) / pow( + n_cells, 1.0 / float(n_param) + ) + param_min.append( + [val for val in np.arange(np.min(idx), np.max(idx), step)] + ) param_max.append( - [val for val in np.arange(np.min(idx) + step, np.max(idx) + step, step)] + [ + val + for val in np.arange( + np.min(idx) + step, np.max(idx) + step, step + ) + ] ) if n_param == 1: diff --git a/src/shapepipe/pipeline/timeout.py b/src/shapepipe/pipeline/timeout.py index d1a63937..f37e1876 100644 --- a/src/shapepipe/pipeline/timeout.py +++ b/src/shapepipe/pipeline/timeout.py @@ -30,7 +30,9 @@ def with_timeout(timeout, log_file): """ def handler(signum, frame): - raise TimeoutError(f"The process time exceeded {timeout}s in {log_file}") + raise TimeoutError( + f"The process time exceeded {timeout}s in {log_file}" + ) def decorator(decorated): diff --git a/src/shapepipe/pipeline/worker_handler.py b/src/shapepipe/pipeline/worker_handler.py index 5950d5b8..bdd8b037 100644 --- a/src/shapepipe/pipeline/worker_handler.py +++ b/src/shapepipe/pipeline/worker_handler.py @@ -145,7 +145,9 @@ def _create_worker_log(self): print(f" - {job_name} PID: {pid} ", end="") - if process_size < self._config.getint("WORKER", "PROCESS_PRINT_LIMIT"): + if process_size < self._config.getint( + "WORKER", "PROCESS_PRINT_LIMIT" + ): print( f'processing {self.worker_dict["file_number_string"]} ' + f'{self.worker_dict["process"]}' @@ -221,8 +223,12 @@ def _log_stdout(self): This method logs the stdout and stderr output of the job. """ - self.w_log.info(f"Process produced the following output: {self._stdout}") + self.w_log.info( + f"Process produced the following output: {self._stdout}" + ) if self._stderr: - self.w_log.info(f"Process produced the following error(s): {self._stderr}") + self.w_log.info( + f"Process produced the following error(s): {self._stderr}" + ) self.worker_dict["stderr"] = True diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index d6ff2ed2..d3cb15b5 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -111,7 +111,9 @@ def close_pipeline_log(self): plur = " was" else: plur = "s were" - final_error_count = f"A total of {self.error_count} error{plur} recorded." + final_error_count = ( + f"A total of {self.error_count} error{plur} recorded." + ) end_text = "Finishing ShapePipe Run" self.log.info(final_error_count) @@ -172,7 +174,9 @@ def _check_dependencies(self): """ module_dep = self._get_module_depends("depends") + [ - dep.split()[0] for dep in requires("ShapePipe") if "extra ==" not in dep + dep.split()[0] + for dep in requires("ShapePipe") + if "extra ==" not in dep ] module_exe = self._get_module_depends("executes") @@ -227,7 +231,9 @@ def _check_module_versions(self): for module in set(self.modules): - module_txt = f" - {module} {self.filehd.module_runners[module].version}" + module_txt = ( + f" - {module} {self.filehd.module_runners[module].version}" + ) self.log.info(module_txt) if self.verbose: @@ -265,7 +271,9 @@ def _get_module_run_methods(self): for module in self.modules: - self.run_method[module] = self.filehd.module_runners[module].run_method + self.run_method[module] = self.filehd.module_runners[ + module + ].run_method def _prep_run(self): """Prepare Run. @@ -409,7 +417,9 @@ def run_mpi(pipe, comm): jobs = split_mpi_jobs(process_list, comm.size) del process_list else: - job_type = module_runner = worker_log = timeout = jobs = run_dirs = None + job_type = module_runner = worker_log = timeout = jobs = ( + run_dirs + ) = None # Broadcast job type to all nodes job_type = comm.bcast(job_type, root=0) @@ -427,7 +437,13 @@ def run_mpi(pipe, comm): # Submit the MPI jobs and gather results results = comm.gather( submit_mpi_jobs( - jobs, config, timeout, run_dirs, module_runner, worker_log, verbose + jobs, + config, + timeout, + run_dirs, + module_runner, + worker_log, + verbose, ), root=0, ) diff --git a/src/shapepipe/tests/test_get_images.py b/src/shapepipe/tests/test_get_images.py index a29ffe2a..860f37f9 100644 --- a/src/shapepipe/tests/test_get_images.py +++ b/src/shapepipe/tests/test_get_images.py @@ -31,9 +31,13 @@ def tearDown(self): def test_in2out_pattern(self): - npt.assert_string_equal(get_images.in2out_pattern(self.number_tile), "123-456") + npt.assert_string_equal( + get_images.in2out_pattern(self.number_tile), "123-456" + ) - npt.assert_string_equal(get_images.in2out_pattern(self.number_exp), "2490092") + npt.assert_string_equal( + get_images.in2out_pattern(self.number_exp), "2490092" + ) npt.assert_raises( TypeError, diff --git a/src/shapepipe/utilities/cfis.py b/src/shapepipe/utilities/cfis.py index af4ae09c..0d94b3be 100644 --- a/src/shapepipe/utilities/cfis.py +++ b/src/shapepipe/utilities/cfis.py @@ -146,7 +146,9 @@ def get_ID(self): else: return f"{m[1]}.{m[2]}" - def print(self, file=sys.stdout, base_name=False, name_only=True, ID_only=False): + def print( + self, file=sys.stdout, base_name=False, name_only=True, ID_only=False + ): """Print. Print image information as ascii Table column. @@ -202,7 +204,10 @@ def print_header(self, file=sys.stdout): output file handle """ - print(f"#Name ra[{unitdef}] dec[{unitdef}] exp_time[s] validation", file=file) + print( + f"#Name ra[{unitdef}] dec[{unitdef}] exp_time[s] validation", + file=file, + ) def log_command(argv, name=None, close_no_return=True): @@ -314,14 +319,17 @@ def my_string_split(string, num=-1, verbose=False, stop=False, sep=None): else: if not string.find(sep): raise ValueError( - f"No separator '{sep}' found in string '{string}', " + "cannot split" + f"No separator '{sep}' found in string '{string}', " + + "cannot split" ) my_sep = sep res = string.split(my_sep) if num != -1 and num != len(res) and stop: - raise CfisError(f"String '{len(res)}' has length {num}, required is {num}") + raise CfisError( + f"String '{len(res)}' has length {num}, required is {num}" + ) return res @@ -557,7 +565,9 @@ def get_tile_number(tile_name): """ m = re.search(r"(\d{3})[\.-](\d{3})", tile_name) if m is None or len(m.groups()) != 2: - raise CfisError(f"Image name '{tile_name}' does not match tile name syntax") + raise CfisError( + f"Image name '{tile_name}' does not match tile name syntax" + ) nix = m.groups()[0] niy = m.groups()[1] @@ -729,7 +739,9 @@ def get_Angle_arr(str_coord, num=-1, wrap=True, verbose=False): array of sky coordinates (pairs ra, dec) """ - angles_mixed = my_string_split(str_coord, num=num, verbose=verbose, stop=True) + angles_mixed = my_string_split( + str_coord, num=num, verbose=verbose, stop=True + ) n = len(angles_mixed) n = int(n / 2) @@ -811,7 +823,9 @@ def create_image_list(fname, ra, dec, exp_time=[], valid=[]): if nf == 0: raise CfisError("No entries in file name list") if (nf != nr or nf != nd) and nr != 0 and nd != 0: - raise CfisError(f"Lists fname, ra, dec have not same length ({nf}, {nr}, {nd})") + raise CfisError( + f"Lists fname, ra, dec have not same length ({nf}, {nr}, {nd})" + ) images = [] for i in range(nf): @@ -835,7 +849,9 @@ def create_image_list(fname, ra, dec, exp_time=[], valid=[]): return images -def get_image_list(inp, band, image_type, col=None, input_format="full", verbose=False): +def get_image_list( + inp, band, image_type, col=None, input_format="full", verbose=False +): """Get Image List. Return list of images. @@ -869,7 +885,9 @@ def get_image_list(inp, band, image_type, col=None, input_format="full", verbose if os.path.isdir(inp): if col is not None: - raise CfisError("Column name (-c option) only valid if input is file") + raise CfisError( + "Column name (-c option) only valid if input is file" + ) # Read file names from directory listing inp_type = "dir" @@ -925,7 +943,9 @@ def get_image_list(inp, band, image_type, col=None, input_format="full", verbose if input_format == "ID_only": pattern = get_file_pattern(r"\d{3}.\d{3}", band, image_type, ext=False) else: - pattern = get_file_pattern(rf"CFIS.\d{{3}}.\d{{3}}\.{band}", band, image_type) + pattern = get_file_pattern( + rf"CFIS.\d{{3}}.\d{{3}}\.{band}", band, image_type + ) for img in image_list: @@ -969,7 +989,13 @@ def exclude(f, exclude_list): def find_image_at_coord( - images, coord, band, image_type, no_cuts=False, input_format="full", verbose=False + images, + coord, + band, + image_type, + no_cuts=False, + input_format="full", + verbose=False, ): """Find Image At Coordinates. @@ -1014,7 +1040,9 @@ def find_image_at_coord( if image_type in ("tile", "weight", "weight.fz"): nix, niy = get_tile_number_from_coord(ra, dec, return_type=int) - tile_name = get_tile_name(nix, niy, band, image_type, input_format=input_format) + tile_name = get_tile_name( + nix, niy, band, image_type, input_format=input_format + ) img_found = [] for img in images: @@ -1069,7 +1097,9 @@ def find_image_at_coord( return img_found -def find_images_in_area(images, angles, band, image_type, no_cuts=False, verbose=False): +def find_images_in_area( + images, angles, band, image_type, no_cuts=False, verbose=False +): """Fine Images In Area. Return image list within coordinate area (rectangle) @@ -1264,7 +1294,9 @@ def plot_area( dy = abs(angles[0].dec - angles[1].dec) dx = getattr(dx, unitdef) dy = getattr(dy, unitdef) - radius = max(dx, dy) / 2 + (size["exposure"] + size["tile"]) * np.sqrt(2) + radius = max(dx, dy) / 2 + (size["exposure"] + size["tile"]) * np.sqrt( + 2 + ) circle = plt.Circle( (ra_c.deg, dec_c.deg), radius, diff --git a/src/shapepipe/utilities/file_system.py b/src/shapepipe/utilities/file_system.py index f657b9c7..603800ee 100644 --- a/src/shapepipe/utilities/file_system.py +++ b/src/shapepipe/utilities/file_system.py @@ -36,7 +36,9 @@ def check_dir(dir_name): """ if not isinstance(dir_name, str): - raise TypeError(f"Directory name must be of type string, not {type(dir_name)}") + raise TypeError( + f"Directory name must be of type string, not {type(dir_name)}" + ) return os.path.isdir(dir_name) @@ -68,4 +70,6 @@ def mkdir(dir_name, check_created=True, exist_ok=True): os.makedirs(dir_name, exist_ok=exist_ok) if check_created and not check_dir(dir_name): - raise FileSystemError(f'Directory "{dir_name}" not found after mkdir command.') + raise FileSystemError( + f'Directory "{dir_name}" not found after mkdir command.' + ) diff --git a/src/shapepipe/utilities/galaxy.py b/src/shapepipe/utilities/galaxy.py index d777b93a..40948a8f 100644 --- a/src/shapepipe/utilities/galaxy.py +++ b/src/shapepipe/utilities/galaxy.py @@ -64,10 +64,14 @@ def sigma_to_fwhm(sigma, pixel_scale=1.0): f"Sigma must be of type numpy array or float, not {type(sigma)}." ) elif isinstance(sigma, np.ndarray) and sigma.dtype != np.float64: - raise TypeError(f"Sigma array values must be of type float, not {sigma.dtype}.") + raise TypeError( + f"Sigma array values must be of type float, not {sigma.dtype}." + ) if not isinstance(pixel_scale, float): - raise TypeError(f"The pixel scale must of type float, not {type(pixel_scale)}.") + raise TypeError( + f"The pixel scale must of type float, not {type(pixel_scale)}." + ) if isinstance(sigma, np.ndarray) and np.any(sigma <= 0.0): raise ValueError( @@ -76,7 +80,8 @@ def sigma_to_fwhm(sigma, pixel_scale=1.0): ) elif isinstance(sigma, float) and sigma <= 0.0: raise ValueError( - f"Invalid standard deviation {sigma}, needs to be greater than " + "0.0." + f"Invalid standard deviation {sigma}, needs to be greater than " + + "0.0." ) if pixel_scale <= 0.0: