From 7ee6bddc057e6ae195753cb1f8748d6af3d3c612 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Thu, 9 Mar 2023 08:50:25 -0500 Subject: [PATCH 01/36] enhance the cube header determinator by checking the spectral dimension too --- spectral_cube/cube_utils.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 07f01fd2..db446c6e 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -778,12 +778,34 @@ def combine_headers(header1, header2, **kwargs): wcs_opt, shape_opt = find_optimal_celestial_wcs([(s1, w1), (s2, w2)], auto_rotate=False, **kwargs) + # find spectral coverage + specw1 = WCS(header1).spectral + specw2 = WCS(header2).spectral + specaxis1 = [x[0] for x in WCS(header1).world_axis_object_components].index('spectral') + specaxis2 = [x[0] for x in WCS(header1).world_axis_object_components].index('spectral') + range1 = specw1.pixel_to_world([0,header1[f'NAXIS{specaxis1+1}']-1]) + range2 = specw2.pixel_to_world([0,header2[f'NAXIS{specaxis1+1}']-1]) + + # check for overlap + # this will raise an exception if the headers are an different units, which we want + if max(range1) < min(range2) or max(range2) < min(range1): + warnings.warn(f"There is no spectral overlap between {range1} and {range2}") + + # check cdelt + dx1 = specw1.proj_plane_pixel_scales()[0] + dx2 = specw2.proj_plane_pixel_scales()[0] + if dx1 != dx2: + raise ValueError(f"Different spectral pixel scale {dx1} vs {dx2}") + + ranges = np.hstack([range1, range2]) + new_naxis = int(np.ceil((ranges.max() - ranges.min()) / np.abs(dx1))) + # Make a new header using the optimal wcs and information from cubes header = header1.copy() header['NAXIS'] = 3 header['NAXIS1'] = shape_opt[1] header['NAXIS2'] = shape_opt[0] - header['NAXIS3'] = header1['NAXIS3'] + header['NAXIS3'] = new_naxis header.update(wcs_opt.to_header()) header['WCSAXES'] = 3 return header From f988d155bf06156e2d28fa75af2dc3a197904faf Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 14:10:15 -0400 Subject: [PATCH 02/36] add option to specify header WIP / stash --- spectral_cube/cube_utils.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index db446c6e..1ccb4951 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -810,7 +810,10 @@ def combine_headers(header1, header2, **kwargs): header['WCSAXES'] = 3 return header -def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, **kwargs): +def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, + target_header=None, + common_beam=None, + **kwargs): ''' This function reprojects cubes onto a common grid and combines them to a single field. @@ -830,32 +833,34 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, **kwa ''' cube1 = cubes[0] - header = cube1.header - # Create a header for a field containing all cubes - for cu in cubes[1:]: - header = combine_headers(header, cu.header, **combine_header_kwargs) + if target_header is None: + target_header = cube1.header + + # Create a header for a field containing all cubes + for cu in cubes[1:]: + target_header = combine_headers(target_header, cu.header, **combine_header_kwargs) # Prepare an array and mask for the final cube - shape_opt = (header['NAXIS3'], header['NAXIS2'], header['NAXIS1']) + shape_opt = (target_header['NAXIS3'], target_header['NAXIS2'], target_header['NAXIS1']) final_array = np.zeros(shape_opt) mask_opt = np.zeros(shape_opt[1:]) for cube in cubes: - # Reproject cubes to the header + # Reproject cubes to the target_header try: if spectral_block_size is not None: - cube_repr = cube.reproject(header, + cube_repr = cube.reproject(target_header, block_size=[spectral_block_size, cube.shape[1], cube.shape[2]], **kwargs) else: - cube_repr = cube.reproject(header, **kwargs) + cube_repr = cube.reproject(target_header, **kwargs) except TypeError: warnings.warn("The block_size argument is not accepted by `reproject`. " "A more recent version may be needed.") - cube_repr = cube.reproject(header, **kwargs) + cube_repr = cube.reproject(target_header, **kwargs) # Create weighting mask (2D) mask = (cube_repr[0:1].get_mask_array()[0]) @@ -874,5 +879,5 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, **kwa final_array[ss] /= mask_opt # Create Cube - cube = cube1.__class__(data=final_array * cube1.unit, wcs=WCS(header)) + cube = cube1.__class__(data=final_array * cube1.unit, wcs=WCS(target_header)) return cube From a9ee1ec6339e4df34602adb08af02e58e4ceae73 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 15:50:22 -0400 Subject: [PATCH 03/36] try to make convolution possible _and_ use memmap'd output --- spectral_cube/cube_utils.py | 43 ++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 1ccb4951..e0c568da 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -812,7 +812,9 @@ def combine_headers(header1, header2, **kwargs): def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, target_header=None, - common_beam=None, + commonbeam=None, + save_to_tmp_dir=True, + use_memmap=True, **kwargs): ''' This function reprojects cubes onto a common grid and combines them to a single field. @@ -826,6 +828,15 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, combine_header_kwargs : dict Keywords passed to `~reproject.mosaicking.find_optimal_celestial_wcs` via `combine_headers`. + commonbeam : Beam + If specified, will smooth the data to this common beam before + reprojecting. + save_to_tmp_dir : bool + Default is to set `save_to_tmp_dir=True` because we expect cubes to be + big. + use_memmap : bool + Use a memory-mapped array to save the mosaicked cube product? + Outputs ------- cube : SpectralCube @@ -843,10 +854,28 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # Prepare an array and mask for the final cube shape_opt = (target_header['NAXIS3'], target_header['NAXIS2'], target_header['NAXIS1']) - final_array = np.zeros(shape_opt) + + + if use_memmap: + ntf = tempfile.NamedTemporaryFile() + final_array = np.memmap(ntf, mode='w+', shape=shape_opt, dtype=float) + else: + final_array = np.zeros(shape_opt) mask_opt = np.zeros(shape_opt[1:]) + # check that the beams are deconvolvable + if commonbeam: + for cube in cubes: + try: + commonbeam.deconvolve(cube.beam) + except radio_beam.BeamError as ex: + raise radio_beam.BeamError("One or more beams could not be " + "deconvolved from the common beam: " + f"{ex}") + for cube in cubes: + if commonbeam: + cube = cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) # Reproject cubes to the target_header try: if spectral_block_size is not None: @@ -854,12 +883,16 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, block_size=[spectral_block_size, cube.shape[1], cube.shape[2]], + save_to_tmp_dir=save_to_tmp_dir, **kwargs) else: - cube_repr = cube.reproject(target_header, **kwargs) - except TypeError: + cube_repr = cube.reproject(target_header, + save_to_tmp_dir=save_to_tmp_dir, + **kwargs) + except TypeError as ex: + # print the exception in case we caught a different TypeError than expected warnings.warn("The block_size argument is not accepted by `reproject`. " - "A more recent version may be needed.") + f"A more recent version may be needed. Exception was: {ex}") cube_repr = cube.reproject(target_header, **kwargs) # Create weighting mask (2D) From 25716935d5de0c969b12dcf8891912fc28c0376d Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 16:20:06 -0400 Subject: [PATCH 04/36] add missing import --- spectral_cube/cube_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index e0c568da..e0810bb4 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -1,5 +1,6 @@ import contextlib import warnings +import tempfile from copy import deepcopy import builtins From a73b1af064e90bb1346958a57da03aa9e8db8fb4 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 16:21:35 -0400 Subject: [PATCH 05/36] need non-array truthiness --- spectral_cube/cube_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index e0810bb4..a69e6d9f 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -865,7 +865,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, mask_opt = np.zeros(shape_opt[1:]) # check that the beams are deconvolvable - if commonbeam: + if commonbeam is not None: for cube in cubes: try: commonbeam.deconvolve(cube.beam) @@ -875,7 +875,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, f"{ex}") for cube in cubes: - if commonbeam: + if commonbeam is not None: cube = cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) # Reproject cubes to the target_header try: From fe92cc97ce7d0b07ca8059e5d5a1be8291adf24d Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 16:22:58 -0400 Subject: [PATCH 06/36] import --- spectral_cube/cube_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index a69e6d9f..53e0ee0d 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -19,7 +19,7 @@ from astropy import units as u import itertools import re -from radio_beam import Beam +from radio_beam import Beam, BeamError def _fix_spectral(wcs): @@ -869,10 +869,10 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, for cube in cubes: try: commonbeam.deconvolve(cube.beam) - except radio_beam.BeamError as ex: - raise radio_beam.BeamError("One or more beams could not be " - "deconvolved from the common beam: " - f"{ex}") + except BeamError as ex: + raise BeamError("One or more beams could not be " + "deconvolved from the common beam: " + f"{ex}") for cube in cubes: if commonbeam is not None: From 94d5dde82ef50812df3f6bebbb1f6f2f35117c86 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 16:26:11 -0400 Subject: [PATCH 07/36] import --- spectral_cube/cube_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 53e0ee0d..bcb52352 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -19,7 +19,8 @@ from astropy import units as u import itertools import re -from radio_beam import Beam, BeamError +from radio_beam import Beam +from radio_beam.utils import BeamError def _fix_spectral(wcs): From eadaa1939dbac472d7bf637a8fdbc91872054405 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 16:33:55 -0400 Subject: [PATCH 08/36] just raise the exception as is, don't wrap it --- spectral_cube/cube_utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index bcb52352..38983269 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -868,12 +868,8 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # check that the beams are deconvolvable if commonbeam is not None: for cube in cubes: - try: - commonbeam.deconvolve(cube.beam) - except BeamError as ex: - raise BeamError("One or more beams could not be " - "deconvolved from the common beam: " - f"{ex}") + # this will raise an exception if any of the cubes have bad beams + commonbeam.deconvolve(cube.beam) for cube in cubes: if commonbeam is not None: From 11aa31c57952454d62a5e99529c622331ec3afb4 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:15:42 -0400 Subject: [PATCH 09/36] refactor cube mosaicing to use reproject.coadd --- spectral_cube/cube_utils.py | 67 ++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 38 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 38983269..358b9823 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -844,6 +844,8 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, cube : SpectralCube A spectral cube with the list of cubes mosaicked together. ''' + from reproject.mosaicking import reproject_and_coadd + from reproject import reproject_interp cube1 = cubes[0] @@ -857,12 +859,14 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # Prepare an array and mask for the final cube shape_opt = (target_header['NAXIS3'], target_header['NAXIS2'], target_header['NAXIS1']) - if use_memmap: ntf = tempfile.NamedTemporaryFile() final_array = np.memmap(ntf, mode='w+', shape=shape_opt, dtype=float) + ntf2 = tempfile.NamedTemporaryFile() + final_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=float) else: final_array = np.zeros(shape_opt) + final_footprint = np.zeros(shape_opt) mask_opt = np.zeros(shape_opt[1:]) # check that the beams are deconvolvable @@ -871,43 +875,30 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # this will raise an exception if any of the cubes have bad beams commonbeam.deconvolve(cube.beam) - for cube in cubes: - if commonbeam is not None: - cube = cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) - # Reproject cubes to the target_header - try: - if spectral_block_size is not None: - cube_repr = cube.reproject(target_header, - block_size=[spectral_block_size, - cube.shape[1], - cube.shape[2]], - save_to_tmp_dir=save_to_tmp_dir, - **kwargs) - else: - cube_repr = cube.reproject(target_header, - save_to_tmp_dir=save_to_tmp_dir, - **kwargs) - except TypeError as ex: - # print the exception in case we caught a different TypeError than expected - warnings.warn("The block_size argument is not accepted by `reproject`. " - f"A more recent version may be needed. Exception was: {ex}") - cube_repr = cube.reproject(target_header, **kwargs) - - # Create weighting mask (2D) - mask = (cube_repr[0:1].get_mask_array()[0]) - mask_opt += mask.astype(float) - - # Go through each slice of the cube, add it to the final array - for ii in range(final_array.shape[0]): - slice1 = np.nan_to_num(cube_repr.unitless_filled_data[ii]) - final_array[ii] = final_array[ii] + slice1 - - # Dividing by the mask throws errors where it is zero - with np.errstate(divide='ignore'): - - # Use weighting mask to average where cubes overlap - for ss in range(final_array.shape[0]): - final_array[ss] /= mask_opt + cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) + for cube in cubes] + + try: + final_array, final_footprint = reproject_and_coadd( + [cube.hdu for cube in cubes], + target_header, + final_array=final_array, + final_footprint=final_footprint, + reproject_function=reproject_interp, + block_size=[(spectral_block_size, cube.shape[1], cube.shape[2]) + for cube in cubes], + ) + except TypeError as ex: + # print the exception in case we caught a different TypeError than expected + warnings.warn("The block_size argument is not accepted by `reproject`. " + f"A more recent version may be needed. Exception was: {ex}") + final_array, final_footprint = reproject_and_coadd( + [cube.hdu for cube in cubes], + target_header, + final_array=final_array, + final_footprint=final_footprint, + reproject_function=reproject_interp, + ) # Create Cube cube = cube1.__class__(data=final_array * cube1.unit, wcs=WCS(target_header)) From 8b26e473272761d44aefa70b5af9272f43e1aa91 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:36:16 -0400 Subject: [PATCH 10/36] refactor to allow for direct FITS output --- spectral_cube/cube_utils.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 358b9823..90f44654 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -817,6 +817,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, commonbeam=None, save_to_tmp_dir=True, use_memmap=True, + output_file=None, **kwargs): ''' This function reprojects cubes onto a common grid and combines them to a single field. @@ -838,6 +839,9 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, big. use_memmap : bool Use a memory-mapped array to save the mosaicked cube product? + output_file : str or None + If specified, this should be a FITS filename that the output *array* + will be stored into (the footprint will not be saved) Outputs ------- @@ -858,12 +862,33 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # Prepare an array and mask for the final cube shape_opt = (target_header['NAXIS3'], target_header['NAXIS2'], target_header['NAXIS1']) - - if use_memmap: + dtype = f"float{target_header['BITPIX']}" + + if output_file is not None: + if not output_file.endswith('.fits'): + raise IOError("Only FITS output is supported") + # https://docs.astropy.org/en/stable/generated/examples/io/skip_create-large-fits.html#sphx-glr-generated-examples-io-skip-create-large-fits-py + hdu = fits.PrimaryHDU(data=np.ones([5,5,5], dtype=dtype), + header=target_header + ) + for kwd in ('NAXIS1', 'NAXIS2', 'NAXIS3'): + hdu.header[kwd] = target_header[kwd] + target_header.tofile(output_file) + with open(output_file, 'rb+') as fobj: + fobj.seek(len(target_header.tostring()) + + (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) + fobj.write(b'\0') + + final_array = fits.getdata(output_file, mode='rb+') + + # use memmap - not a FITS file - for the footprint + ntf2 = tempfile.NamedTemporaryFile() + final_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) + elif use_memmap: ntf = tempfile.NamedTemporaryFile() - final_array = np.memmap(ntf, mode='w+', shape=shape_opt, dtype=float) + final_array = np.memmap(ntf, mode='w+', shape=shape_opt, dtype=dtype) ntf2 = tempfile.NamedTemporaryFile() - final_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=float) + final_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) else: final_array = np.zeros(shape_opt) final_footprint = np.zeros(shape_opt) From d2929f80a427fb9ccf534b3ad6f8442ddad8010a Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:39:20 -0400 Subject: [PATCH 11/36] bitpix dtype --- spectral_cube/cube_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 90f44654..bf1b5b7c 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -862,7 +862,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # Prepare an array and mask for the final cube shape_opt = (target_header['NAXIS3'], target_header['NAXIS2'], target_header['NAXIS1']) - dtype = f"float{target_header['BITPIX']}" + dtype = f"float{int(abs(target_header['BITPIX']))}" if output_file is not None: if not output_file.endswith('.fits'): From 66657033f1221e5963958af2ba94787f5130a191 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:46:25 -0400 Subject: [PATCH 12/36] getdata doesn't support mode --- spectral_cube/cube_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index bf1b5b7c..2b288d70 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -879,7 +879,8 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) fobj.write(b'\0') - final_array = fits.getdata(output_file, mode='rb+') + hdu = fits.open(output_file, mode='rb+') + final_array = hdu.data # use memmap - not a FITS file - for the footprint ntf2 = tempfile.NamedTemporaryFile() From fe5fbb9853e688a185fa100557802c4ef023b466 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:47:02 -0400 Subject: [PATCH 13/36] force overwrite --- spectral_cube/cube_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 2b288d70..d39ad8f3 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -879,7 +879,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) fobj.write(b'\0') - hdu = fits.open(output_file, mode='rb+') + hdu = fits.open(output_file, mode='rb+', overwrite=True) final_array = hdu.data # use memmap - not a FITS file - for the footprint From 1d6231425aeed59adce87fd28dad68b14ecd67c5 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:47:48 -0400 Subject: [PATCH 14/36] force overwrite --- spectral_cube/cube_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index d39ad8f3..5e5b7907 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -873,7 +873,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, ) for kwd in ('NAXIS1', 'NAXIS2', 'NAXIS3'): hdu.header[kwd] = target_header[kwd] - target_header.tofile(output_file) + target_header.tofile(output_file, overwrite=True) with open(output_file, 'rb+') as fobj: fobj.seek(len(target_header.tostring()) + (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) From ccd1d696796fcaa60f05a87d39ebc9e7b8049283 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:48:26 -0400 Subject: [PATCH 15/36] mode for fits --- spectral_cube/cube_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 5e5b7907..18122079 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -879,7 +879,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) fobj.write(b'\0') - hdu = fits.open(output_file, mode='rb+', overwrite=True) + hdu = fits.open(output_file, mode='update', overwrite=True) final_array = hdu.data # use memmap - not a FITS file - for the footprint From ca73b575cafbcd3a269c1f3316c6080ade1b4e89 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 17:49:15 -0400 Subject: [PATCH 16/36] hdu0 --- spectral_cube/cube_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 18122079..771a3656 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -880,7 +880,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, fobj.write(b'\0') hdu = fits.open(output_file, mode='update', overwrite=True) - final_array = hdu.data + final_array = hdu[0].data # use memmap - not a FITS file - for the footprint ntf2 = tempfile.NamedTemporaryFile() From 803daa00b28274574fb7ccd471bdd30250358a7e Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 18:10:55 -0400 Subject: [PATCH 17/36] refactor to use output_array --- spectral_cube/cube_utils.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 771a3656..5e310e28 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -880,19 +880,19 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, fobj.write(b'\0') hdu = fits.open(output_file, mode='update', overwrite=True) - final_array = hdu[0].data + output_array = hdu[0].data # use memmap - not a FITS file - for the footprint ntf2 = tempfile.NamedTemporaryFile() - final_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) + output_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) elif use_memmap: ntf = tempfile.NamedTemporaryFile() - final_array = np.memmap(ntf, mode='w+', shape=shape_opt, dtype=dtype) + output_array = np.memmap(ntf, mode='w+', shape=shape_opt, dtype=dtype) ntf2 = tempfile.NamedTemporaryFile() - final_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) + output_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) else: - final_array = np.zeros(shape_opt) - final_footprint = np.zeros(shape_opt) + output_array = np.zeros(shape_opt) + output_footprint = np.zeros(shape_opt) mask_opt = np.zeros(shape_opt[1:]) # check that the beams are deconvolvable @@ -905,11 +905,11 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, for cube in cubes] try: - final_array, final_footprint = reproject_and_coadd( + output_array, output_footprint = reproject_and_coadd( [cube.hdu for cube in cubes], target_header, - final_array=final_array, - final_footprint=final_footprint, + output_array=output_array, + output_footprint=output_footprint, reproject_function=reproject_interp, block_size=[(spectral_block_size, cube.shape[1], cube.shape[2]) for cube in cubes], @@ -918,14 +918,14 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # print the exception in case we caught a different TypeError than expected warnings.warn("The block_size argument is not accepted by `reproject`. " f"A more recent version may be needed. Exception was: {ex}") - final_array, final_footprint = reproject_and_coadd( + output_array, output_footprint = reproject_and_coadd( [cube.hdu for cube in cubes], target_header, - final_array=final_array, - final_footprint=final_footprint, + output_array=output_array, + output_footprint=output_footprint, reproject_function=reproject_interp, ) # Create Cube - cube = cube1.__class__(data=final_array * cube1.unit, wcs=WCS(target_header)) + cube = cube1.__class__(data=output_array * cube1.unit, wcs=WCS(target_header)) return cube From a1df3e2046bd74c2797fad29bdd7f50829775aeb Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 18:22:23 -0400 Subject: [PATCH 18/36] allow setting no spectral block size --- spectral_cube/cube_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 5e310e28..232ff4fb 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -911,8 +911,9 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, output_array=output_array, output_footprint=output_footprint, reproject_function=reproject_interp, - block_size=[(spectral_block_size, cube.shape[1], cube.shape[2]) - for cube in cubes], + block_size=(None if spectral_block_size is None else + [(spectral_block_size, cube.shape[1], cube.shape[2]) + for cube in cubes]), ) except TypeError as ex: # print the exception in case we caught a different TypeError than expected From 7b39880950471f6e94077ba62aabc3a3132837a0 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 18:42:04 -0400 Subject: [PATCH 19/36] add flush --- spectral_cube/cube_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 232ff4fb..30be00ad 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -929,4 +929,9 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # Create Cube cube = cube1.__class__(data=output_array * cube1.unit, wcs=WCS(target_header)) + + if output_file is not None: + hdu.flush() + hdu.close() + return cube From aaba58a78874673dbafb2345bdeb5cdab3519ad4 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 19:20:15 -0400 Subject: [PATCH 20/36] need to flush occasionally --- spectral_cube/cube_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 30be00ad..449bc9ef 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -881,6 +881,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, hdu = fits.open(output_file, mode='update', overwrite=True) output_array = hdu[0].data + hdu.flush() # make sure the header gets written right # use memmap - not a FITS file - for the footprint ntf2 = tempfile.NamedTemporaryFile() From c231f0aaf86876540c18c9576c7e2d037e9b46bd Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 21:34:02 -0400 Subject: [PATCH 21/36] allow channel-by-channel mosaicing instead of cube-by-cube --- spectral_cube/cube_utils.py | 81 ++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 449bc9ef..60ec6bc1 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -818,6 +818,8 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, save_to_tmp_dir=True, use_memmap=True, output_file=None, + method='cube', + verbose=True, **kwargs): ''' This function reprojects cubes onto a common grid and combines them to a single field. @@ -842,6 +844,11 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, output_file : str or None If specified, this should be a FITS filename that the output *array* will be stored into (the footprint will not be saved) + method : 'cube' or 'channel' + Over what dimension should we iterate? Options are 'cube' and + 'channel'. + verbose : bool + Progressbars? Outputs ------- @@ -851,6 +858,11 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, from reproject.mosaicking import reproject_and_coadd from reproject import reproject_interp + if verbose: + from tqdm import tqdm + else: + tqdm = lambda x: x + cube1 = cubes[0] if target_header is None: @@ -902,31 +914,52 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # this will raise an exception if any of the cubes have bad beams commonbeam.deconvolve(cube.beam) - cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) - for cube in cubes] + cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir and method == 'cube') + for cube in tqdm(cubes)] - try: - output_array, output_footprint = reproject_and_coadd( - [cube.hdu for cube in cubes], - target_header, - output_array=output_array, - output_footprint=output_footprint, - reproject_function=reproject_interp, - block_size=(None if spectral_block_size is None else - [(spectral_block_size, cube.shape[1], cube.shape[2]) - for cube in cubes]), - ) - except TypeError as ex: - # print the exception in case we caught a different TypeError than expected - warnings.warn("The block_size argument is not accepted by `reproject`. " - f"A more recent version may be needed. Exception was: {ex}") - output_array, output_footprint = reproject_and_coadd( - [cube.hdu for cube in cubes], - target_header, - output_array=output_array, - output_footprint=output_footprint, - reproject_function=reproject_interp, - ) + if method == 'cube': + try: + output_array, output_footprint = reproject_and_coadd( + [cube.hdu for cube in cubes], + target_header, + output_array=output_array, + output_footprint=output_footprint, + reproject_function=reproject_interp, + block_size=(None if spectral_block_size is None else + [(spectral_block_size, cube.shape[1], cube.shape[2]) + for cube in cubes]), + ) + except TypeError as ex: + # print the exception in case we caught a different TypeError than expected + warnings.warn("The block_size argument is not accepted by `reproject`. " + f"A more recent version may be needed. Exception was: {ex}") + output_array, output_footprint = reproject_and_coadd( + [cube.hdu for cube in cubes], + target_header, + output_array=output_array, + output_footprint=output_footprint, + reproject_function=reproject_interp, + ) + elif method == 'channel': + outwcs = WCS(target_header) + channels = outwcs.spectral.pixel_to_world(np.arange(target_header['NAXIS3'])) + dx = channels[1] - channels[0] + + for ii, channel in tqdm(enumerate(channels)): + + # grab +/- 0.5 channel to enable interpolation + # ideally this should produce 2-channel cubes, but there are corner cases + # where it will get 3+ channels, which is inefficient but kinda harmless + scubes = [cube.spectral_slab(channel - dx, channel + dx) + for cube in cubes] + + output_array, output_footprint = reproject_and_coadd( + [cube.hdu for cube in cubes], + target_header, + output_array=output_array[ii,:,:], + output_footprint=output_footprint[ii,:,:], + reproject_function=reproject_interp, + ) # Create Cube cube = cube1.__class__(data=output_array * cube1.unit, wcs=WCS(target_header)) From 5385daebdf95b8c2da330d7b9d682fc9dcf5d277 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 21:53:10 -0400 Subject: [PATCH 22/36] add progressbar & frequent flush callback --- spectral_cube/cube_utils.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 60ec6bc1..dde190ad 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -22,6 +22,7 @@ from radio_beam import Beam from radio_beam.utils import BeamError +from functools import partial def _fix_spectral(wcs): """ @@ -859,9 +860,9 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, from reproject import reproject_interp if verbose: - from tqdm import tqdm + from tqdm import tqdm as std_tqdm else: - tqdm = lambda x: x + std_tqdm = lambda x: x cube1 = cubes[0] @@ -915,7 +916,12 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, commonbeam.deconvolve(cube.beam) cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir and method == 'cube') - for cube in tqdm(cubes)] + for cube in std_tqdm(cubes)] + + class tqdm(std_tqdm): + def update(self, n=1): + hdu.flush() + super().update(n) if method == 'cube': try: @@ -925,6 +931,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, output_array=output_array, output_footprint=output_footprint, reproject_function=reproject_interp, + progressbar=tqdm if verbose else False, block_size=(None if spectral_block_size is None else [(spectral_block_size, cube.shape[1], cube.shape[2]) for cube in cubes]), @@ -939,12 +946,14 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, output_array=output_array, output_footprint=output_footprint, reproject_function=reproject_interp, + progressbar=tqdm if verbose else False, ) elif method == 'channel': outwcs = WCS(target_header) channels = outwcs.spectral.pixel_to_world(np.arange(target_header['NAXIS3'])) dx = channels[1] - channels[0] + for ii, channel in tqdm(enumerate(channels)): # grab +/- 0.5 channel to enable interpolation @@ -953,12 +962,15 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, scubes = [cube.spectral_slab(channel - dx, channel + dx) for cube in cubes] - output_array, output_footprint = reproject_and_coadd( + # project into array w/"dummy" third dimension + output_array_, output_footprint_ = reproject_and_coadd( [cube.hdu for cube in cubes], - target_header, - output_array=output_array[ii,:,:], - output_footprint=output_footprint[ii,:,:], + outwcs[ii:ii+1, :, :], + shape_out=(1,) + output_array.shape[1:], + output_array=output_array[ii:ii+1,:,:], + output_footprint=output_footprint[ii:ii+1,:,:], reproject_function=reproject_interp, + progressbar=tqdm if verbose else False, ) # Create Cube From fae18376617d67dc5540cf27603ad7fbe1dce01c Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 14 Mar 2023 11:07:00 -0400 Subject: [PATCH 23/36] fix some critical typos and enable resuming from partly-written file (but it doesn't work right yet) --- spectral_cube/cube_utils.py | 44 +++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index dde190ad..9c4ae55c 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -1,6 +1,7 @@ import contextlib import warnings import tempfile +import os from copy import deepcopy import builtins @@ -858,6 +859,9 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, ''' from reproject.mosaicking import reproject_and_coadd from reproject import reproject_interp + import warnings + from astropy.utils.exceptions import AstropyUserWarning + warnings.filterwarnings('ignore', category=AstropyUserWarning) if verbose: from tqdm import tqdm as std_tqdm @@ -880,23 +884,26 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, if output_file is not None: if not output_file.endswith('.fits'): raise IOError("Only FITS output is supported") - # https://docs.astropy.org/en/stable/generated/examples/io/skip_create-large-fits.html#sphx-glr-generated-examples-io-skip-create-large-fits-py - hdu = fits.PrimaryHDU(data=np.ones([5,5,5], dtype=dtype), - header=target_header - ) - for kwd in ('NAXIS1', 'NAXIS2', 'NAXIS3'): - hdu.header[kwd] = target_header[kwd] - target_header.tofile(output_file, overwrite=True) - with open(output_file, 'rb+') as fobj: - fobj.seek(len(target_header.tostring()) + - (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) - fobj.write(b'\0') + if not os.path.exists(output_file): + # https://docs.astropy.org/en/stable/generated/examples/io/skip_create-large-fits.html#sphx-glr-generated-examples-io-skip-create-large-fits-py + hdu = fits.PrimaryHDU(data=np.ones([5,5,5], dtype=dtype), + header=target_header + ) + for kwd in ('NAXIS1', 'NAXIS2', 'NAXIS3'): + hdu.header[kwd] = target_header[kwd] + + target_header.tofile(output_file, overwrite=True) + with open(output_file, 'rb+') as fobj: + fobj.seek(len(target_header.tostring()) + + (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) + fobj.write(b'\0') hdu = fits.open(output_file, mode='update', overwrite=True) output_array = hdu[0].data hdu.flush() # make sure the header gets written right # use memmap - not a FITS file - for the footprint + # if we want to do partial writing, though, it's best to make footprint an extension ntf2 = tempfile.NamedTemporaryFile() output_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) elif use_memmap: @@ -915,15 +922,16 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, # this will raise an exception if any of the cubes have bad beams commonbeam.deconvolve(cube.beam) - cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir and method == 'cube') - for cube in std_tqdm(cubes)] - class tqdm(std_tqdm): def update(self, n=1): hdu.flush() super().update(n) if method == 'cube': + + cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) + for cube in std_tqdm(cubes)] + try: output_array, output_footprint = reproject_and_coadd( [cube.hdu for cube in cubes], @@ -952,6 +960,8 @@ def update(self, n=1): outwcs = WCS(target_header) channels = outwcs.spectral.pixel_to_world(np.arange(target_header['NAXIS3'])) dx = channels[1] - channels[0] + if verbose: + print(f"Channel mode: dx={dx}. Looping over {len(channels)} channels and {len(cubes)} cubes") for ii, channel in tqdm(enumerate(channels)): @@ -959,12 +969,14 @@ def update(self, n=1): # grab +/- 0.5 channel to enable interpolation # ideally this should produce 2-channel cubes, but there are corner cases # where it will get 3+ channels, which is inefficient but kinda harmless - scubes = [cube.spectral_slab(channel - dx, channel + dx) + scubes = [(cube.spectral_slab(channel - dx, channel + dx) + .minimal_subcube() + .convolve_to(commonbeam)) for cube in cubes] # project into array w/"dummy" third dimension output_array_, output_footprint_ = reproject_and_coadd( - [cube.hdu for cube in cubes], + [cube.hdu for cube in scubes], outwcs[ii:ii+1, :, :], shape_out=(1,) + output_array.shape[1:], output_array=output_array[ii:ii+1,:,:], From ba2d131748330b1b665c350902b5f9ea65f96477 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 14 Mar 2023 16:08:30 -0400 Subject: [PATCH 24/36] restructure a bunch of things that are being done too expensively and add a lot of verbosity --- spectral_cube/cube_utils.py | 90 ++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 9c4ae55c..fae304fa 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -2,6 +2,7 @@ import warnings import tempfile import os +import time from copy import deepcopy import builtins @@ -868,6 +869,12 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, else: std_tqdm = lambda x: x + def log_(x): + if verbose: + log.info(x) + else: + log.debug(x) + cube1 = cubes[0] if target_header is None: @@ -882,6 +889,8 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, dtype = f"float{int(abs(target_header['BITPIX']))}" if output_file is not None: + log_(f"Using output file {output_file}") + t0 = time.time() if not output_file.endswith('.fits'): raise IOError("Only FITS output is supported") if not os.path.exists(output_file): @@ -892,26 +901,37 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, for kwd in ('NAXIS1', 'NAXIS2', 'NAXIS3'): hdu.header[kwd] = target_header[kwd] + log_(f"Dumping header to file {output_file} (dt={time.time()-t0})") target_header.tofile(output_file, overwrite=True) with open(output_file, 'rb+') as fobj: fobj.seek(len(target_header.tostring()) + (np.prod(shape_opt) * np.abs(target_header['BITPIX']//8)) - 1) fobj.write(b'\0') + log_(f"Loading header from file {output_file} (dt={time.time()-t0})") hdu = fits.open(output_file, mode='update', overwrite=True) output_array = hdu[0].data hdu.flush() # make sure the header gets written right + log_(f"Creating footprint file dt={time.time()-t0}") # use memmap - not a FITS file - for the footprint # if we want to do partial writing, though, it's best to make footprint an extension ntf2 = tempfile.NamedTemporaryFile() output_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) + + # default footprint to 1 assuming there is some stuff already in the image + # this is a hack and maybe just shouldn't be attempted + #log_("Initializing footprint to 1s") + #output_footprint[:] = 1 # takes an hour?! + log_(f"Done initializing memory dt={time.time()-t0}") elif use_memmap: + log_("Using memmap") ntf = tempfile.NamedTemporaryFile() output_array = np.memmap(ntf, mode='w+', shape=shape_opt, dtype=dtype) ntf2 = tempfile.NamedTemporaryFile() output_footprint = np.memmap(ntf2, mode='w+', shape=shape_opt, dtype=dtype) else: + log_("Using memory") output_array = np.zeros(shape_opt) output_footprint = np.zeros(shape_opt) mask_opt = np.zeros(shape_opt[1:]) @@ -928,9 +948,10 @@ def update(self, n=1): super().update(n) if method == 'cube': + log_("Using Cube method") cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) - for cube in std_tqdm(cubes)] + for cube in std_tqdm(cubes, desc="Convolve:")] try: output_array, output_footprint = reproject_and_coadd( @@ -957,33 +978,52 @@ def update(self, n=1): progressbar=tqdm if verbose else False, ) elif method == 'channel': + log_("Using Channel method") outwcs = WCS(target_header) channels = outwcs.spectral.pixel_to_world(np.arange(target_header['NAXIS3'])) dx = channels[1] - channels[0] - if verbose: - print(f"Channel mode: dx={dx}. Looping over {len(channels)} channels and {len(cubes)} cubes") - - - for ii, channel in tqdm(enumerate(channels)): - - # grab +/- 0.5 channel to enable interpolation - # ideally this should produce 2-channel cubes, but there are corner cases - # where it will get 3+ channels, which is inefficient but kinda harmless - scubes = [(cube.spectral_slab(channel - dx, channel + dx) - .minimal_subcube() - .convolve_to(commonbeam)) - for cube in cubes] - - # project into array w/"dummy" third dimension - output_array_, output_footprint_ = reproject_and_coadd( - [cube.hdu for cube in scubes], - outwcs[ii:ii+1, :, :], - shape_out=(1,) + output_array.shape[1:], - output_array=output_array[ii:ii+1,:,:], - output_footprint=output_footprint[ii:ii+1,:,:], - reproject_function=reproject_interp, - progressbar=tqdm if verbose else False, - ) + log_(f"Channel mode: dx={dx}. Looping over {len(channels)} channels and {len(cubes)} cubes") + + mincube_slices = [cube[cube.shape[0]//2:cube.shape[0]//2+1] + .subcube_slices_from_mask(cube[cube.shape[0]//2:cube.shape[0]//2+1].mask, + spatial_only=True) + for cube in std_tqdm(cubes, desc='MinSubSlices:')] + + for ii, channel in tqdm(enumerate(channels), desc="Channels:"): + + # grab a 2-channel slab + # this is very verbose but quite simple & cheap + # going to spectral_slab(channel-dx, channel+dx) gives 3-pixel cubes most often, + # which results in a 50% overhead in smoothing, etc. + chans = [(cube.closest_spectral_channel(channel) if cube.spectral_axis[cube.closest_spectral_channel(channel)] < channel else cube.closest_spectral_channel(channel)+1, + cube.closest_spectral_channel(channel) if cube.spectral_axis[cube.closest_spectral_channel(channel)] > channel else cube.closest_spectral_channel(channel)-1) + for cube in std_tqdm(cubes, delay=5, desc='ChanSel:')] + # reversed spectral axes still break things + # and we want two channels width, not one + chans = [(ch1, ch2+1) if ch1 < ch2 else (ch2, ch1+1) for ch1, ch2 in chans] + with warnings.catch_warnings(): + warnings.simplefilter('ignore') # seriously NO WARNINGS. + + scubes = [(cube[ch1:ch2, slices[1], slices[2]] + .convolve_to(commonbeam) + .rechunk()) + for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes), + delay=5, desc='Subcubes:')] + + # reproject_and_coadd requires the actual arrays, so this is the convolution step + hdus = [(cube.unitless_filled_data[:], cube.wcs) + for cube in std_tqdm(scubes, delay=5, desc='Data:')] + + # project into array w/"dummy" third dimension + output_array_, output_footprint_ = reproject_and_coadd( + hdus, + outwcs[ii:ii+1, :, :], + shape_out=(1,) + output_array.shape[1:], + output_array=output_array[ii:ii+1,:,:], + output_footprint=output_footprint[ii:ii+1,:,:], + reproject_function=reproject_interp, + progressbar=tqdm if verbose else False, + ) # Create Cube cube = cube1.__class__(data=output_array * cube1.unit, wcs=WCS(target_header)) From 70155bc7ba2c365290486bae4efc1c41e69edc4a Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 14 Mar 2023 18:47:00 -0400 Subject: [PATCH 25/36] more verbosity & parallelism --- spectral_cube/cube_utils.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index fae304fa..00d1f2b4 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -8,6 +8,7 @@ import builtins import dask.array as da +from dask.distributed import Client import numpy as np from astropy.wcs.utils import proj_plane_pixel_area from astropy.wcs import (WCSSUB_SPECTRAL, WCSSUB_LONGITUDE, WCSSUB_LATITUDE) @@ -23,6 +24,7 @@ import re from radio_beam import Beam from radio_beam.utils import BeamError +from multiprocessing import Process, Pool from functools import partial @@ -815,6 +817,12 @@ def combine_headers(header1, header2, **kwargs): header['WCSAXES'] = 3 return header +def _getdata(cube): + """ + Must be defined out-of-scope to enable pickling + """ + return (cube.unitless_filled_data[:], cube.wcs) + def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, target_header=None, commonbeam=None, @@ -981,7 +989,7 @@ def update(self, n=1): log_("Using Channel method") outwcs = WCS(target_header) channels = outwcs.spectral.pixel_to_world(np.arange(target_header['NAXIS3'])) - dx = channels[1] - channels[0] + dx = outwcs.spectral.proj_plane_pixel_scales()[0] log_(f"Channel mode: dx={dx}. Looping over {len(channels)} channels and {len(cubes)} cubes") mincube_slices = [cube[cube.shape[0]//2:cube.shape[0]//2+1] @@ -989,7 +997,10 @@ def update(self, n=1): spatial_only=True) for cube in std_tqdm(cubes, desc='MinSubSlices:')] - for ii, channel in tqdm(enumerate(channels), desc="Channels:"): + pbar = tqdm(enumerate(channels), desc="Channels") + for ii, channel in pbar: + pbar.set_description(f"Channel {ii}={channel}") + print() # grab a 2-channel slab # this is very verbose but quite simple & cheap @@ -1008,11 +1019,18 @@ def update(self, n=1): .convolve_to(commonbeam) .rechunk()) for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes), - delay=5, desc='Subcubes:')] + delay=5, desc='Subcubes')] + print() # reproject_and_coadd requires the actual arrays, so this is the convolution step - hdus = [(cube.unitless_filled_data[:], cube.wcs) - for cube in std_tqdm(scubes, delay=5, desc='Data:')] + #hdus = [(cube._get_filled_data(), cube.wcs) + # for cube in std_tqdm(scubes, delay=5, desc='Data/conv')] + + datas = [cube._get_filled_data() for cube in scubes] + wcses = [cube.wcs for cube in scubes] + with Client() as client: + datas = client.gather(datas) + hdus = list(zip(datas, wcses)) # project into array w/"dummy" third dimension output_array_, output_footprint_ = reproject_and_coadd( @@ -1022,9 +1040,11 @@ def update(self, n=1): output_array=output_array[ii:ii+1,:,:], output_footprint=output_footprint[ii:ii+1,:,:], reproject_function=reproject_interp, - progressbar=tqdm if verbose else False, + progressbar=partial(tqdm, desc='coadd') if verbose else False, ) + pbar.set_description(f"Channel {ii}={channel} done") + # Create Cube cube = cube1.__class__(data=output_array * cube1.unit, wcs=WCS(target_header)) From 5ff9f33a11e7c0aa3c30169d63a8afcab1de5c04 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 15 Mar 2023 11:17:59 -0400 Subject: [PATCH 26/36] this is very verbose now, but still trying to make verbosity flexible --- spectral_cube/cube_utils.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 00d1f2b4..f58865d8 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -875,7 +875,16 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, if verbose: from tqdm import tqdm as std_tqdm else: - std_tqdm = lambda x: x + class tqdm: + def __init__(self, x): + return x + def __call__(self, x) + return x + def set_description(self, **kwargs): + pass + def update(self, **kwargs): + pass + std_tqdm = tqdm def log_(x): if verbose: @@ -950,10 +959,11 @@ def log_(x): # this will raise an exception if any of the cubes have bad beams commonbeam.deconvolve(cube.beam) - class tqdm(std_tqdm): - def update(self, n=1): - hdu.flush() - super().update(n) + if verbose: + class tqdm(std_tqdm): + def update(self, n=1): + hdu.flush() # write to disk on each iteration + super().update(n) if method == 'cube': log_("Using Cube method") @@ -1000,7 +1010,6 @@ def update(self, n=1): pbar = tqdm(enumerate(channels), desc="Channels") for ii, channel in pbar: pbar.set_description(f"Channel {ii}={channel}") - print() # grab a 2-channel slab # this is very verbose but quite simple & cheap @@ -1020,7 +1029,7 @@ def update(self, n=1): .rechunk()) for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes), delay=5, desc='Subcubes')] - print() + # reproject_and_coadd requires the actual arrays, so this is the convolution step #hdus = [(cube._get_filled_data(), cube.wcs) From a9adc6cd22faf1fd0762925e40a1aecdb40f8aac Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 15 Mar 2023 11:26:41 -0400 Subject: [PATCH 27/36] allow weights to be included --- spectral_cube/cube_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index f58865d8..11b350ff 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -826,6 +826,7 @@ def _getdata(cube): def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, target_header=None, commonbeam=None, + weightcubes=None, save_to_tmp_dir=True, use_memmap=True, output_file=None, @@ -847,6 +848,8 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, commonbeam : Beam If specified, will smooth the data to this common beam before reprojecting. + weightcubes : None + Cubes with same shape as input cubes containing the weights save_to_tmp_dir : bool Default is to set `save_to_tmp_dir=True` because we expect cubes to be big. @@ -975,6 +978,7 @@ def update(self, n=1): output_array, output_footprint = reproject_and_coadd( [cube.hdu for cube in cubes], target_header, + weights_in=[cube.hdu for cube in weightcubes] if weightcubes is None else None, output_array=output_array, output_footprint=output_footprint, reproject_function=reproject_interp, @@ -990,6 +994,7 @@ def update(self, n=1): output_array, output_footprint = reproject_and_coadd( [cube.hdu for cube in cubes], target_header, + weights_in=[cube.hdu for cube in weightcubes] if weightcubes is None else None, output_array=output_array, output_footprint=output_footprint, reproject_function=reproject_interp, @@ -1049,6 +1054,7 @@ def update(self, n=1): output_array=output_array[ii:ii+1,:,:], output_footprint=output_footprint[ii:ii+1,:,:], reproject_function=reproject_interp, + weights_in=weightcubes[ii:ii+1].hdu if weightcubes is not None else None, progressbar=partial(tqdm, desc='coadd') if verbose else False, ) From 7f1e86600b15e84983d73d96f1aa41cb6a5e9b5c Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Wed, 15 Mar 2023 11:30:59 -0400 Subject: [PATCH 28/36] typo --- spectral_cube/cube_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 11b350ff..36b32f18 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -881,7 +881,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, class tqdm: def __init__(self, x): return x - def __call__(self, x) + def __call__(self, x): return x def set_description(self, **kwargs): pass From 8c285e7f2e5f534b74e9bdbfefc2d76268b79109 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 20 Mar 2023 13:18:00 -0400 Subject: [PATCH 29/36] add explanations of some choices --- spectral_cube/cube_utils.py | 44 ++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 36b32f18..3b9e6b79 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -956,11 +956,16 @@ def log_(x): output_footprint = np.zeros(shape_opt) mask_opt = np.zeros(shape_opt[1:]) + # check that the beams are deconvolvable if commonbeam is not None: - for cube in cubes: + # assemble beams + beams = [cube.beam if hasattr(cube, 'beam') else cube.beams.common_beam() + for cube in cubes] + + for beam in beams: # this will raise an exception if any of the cubes have bad beams - commonbeam.deconvolve(cube.beam) + commonbeam.deconvolve(beam) if verbose: class tqdm(std_tqdm): @@ -970,15 +975,18 @@ def update(self, n=1): if method == 'cube': log_("Using Cube method") + # Cube method: Regrid the whole cube in one operation. + # Let reproject_and_coadd handle any iterations - cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) - for cube in std_tqdm(cubes, desc="Convolve:")] + if commonbeam is not None: + cubes = [cube.convolve_to(commonbeam, save_to_tmp_dir=save_to_tmp_dir) + for cube in std_tqdm(cubes, desc="Convolve:")] try: output_array, output_footprint = reproject_and_coadd( [cube.hdu for cube in cubes], target_header, - weights_in=[cube.hdu for cube in weightcubes] if weightcubes is None else None, + input_weights=[cube.hdu for cube in weightcubes] if weightcubes is None else None, output_array=output_array, output_footprint=output_footprint, reproject_function=reproject_interp, @@ -994,7 +1002,7 @@ def update(self, n=1): output_array, output_footprint = reproject_and_coadd( [cube.hdu for cube in cubes], target_header, - weights_in=[cube.hdu for cube in weightcubes] if weightcubes is None else None, + input_weights=[cube.hdu for cube in weightcubes] if weightcubes is None else None, output_array=output_array, output_footprint=output_footprint, reproject_function=reproject_interp, @@ -1002,6 +1010,13 @@ def update(self, n=1): ) elif method == 'channel': log_("Using Channel method") + # Channel method: manually downselect to go channel-by-channel in the + # input cubes before handing off material to reproject_and_coadd This + # approach allows us more direct & granular control over memory and is + # likely better for large-area cubes + # (ideally we'd let Dask handle all the memory allocation choices under + # the hood, but as of early 2023, we do not yet have that capability) + outwcs = WCS(target_header) channels = outwcs.spectral.pixel_to_world(np.arange(target_header['NAXIS3'])) dx = outwcs.spectral.proj_plane_pixel_scales()[0] @@ -1010,7 +1025,7 @@ def update(self, n=1): mincube_slices = [cube[cube.shape[0]//2:cube.shape[0]//2+1] .subcube_slices_from_mask(cube[cube.shape[0]//2:cube.shape[0]//2+1].mask, spatial_only=True) - for cube in std_tqdm(cubes, desc='MinSubSlices:')] + for cube in std_tqdm(cubes, desc='MinSubSlices:', delay=5)] pbar = tqdm(enumerate(channels), desc="Channels") for ii, channel in pbar: @@ -1035,11 +1050,23 @@ def update(self, n=1): for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes), delay=5, desc='Subcubes')] + if weightcubes is not None: + sweightcubes = [cube[ch1:ch2, slices[1], slices[2]] + for (ch1, ch2), slices, cube + in std_tqdm(zip(chans, mincube_slices, weightcubes), + delay=5, desc='Subweight')] + # reproject_and_coadd requires the actual arrays, so this is the convolution step + + # commented out approach here: just let spectral-cube handle the convolution etc. #hdus = [(cube._get_filled_data(), cube.wcs) # for cube in std_tqdm(scubes, delay=5, desc='Data/conv')] + # somewhat faster (?) version - ask the dask client to handle + # gathering the data + # (this version is capable of parallelizing over many cubes, in + # theory; the previous would treat each cube in serial) datas = [cube._get_filled_data() for cube in scubes] wcses = [cube.wcs for cube in scubes] with Client() as client: @@ -1047,6 +1074,7 @@ def update(self, n=1): hdus = list(zip(datas, wcses)) # project into array w/"dummy" third dimension + # (outputs are not used; data is written directly into the output array chunks) output_array_, output_footprint_ = reproject_and_coadd( hdus, outwcs[ii:ii+1, :, :], @@ -1054,7 +1082,7 @@ def update(self, n=1): output_array=output_array[ii:ii+1,:,:], output_footprint=output_footprint[ii:ii+1,:,:], reproject_function=reproject_interp, - weights_in=weightcubes[ii:ii+1].hdu if weightcubes is not None else None, + input_weights=sweightcubes[ii:ii+1].hdu if weightcubes is not None else None, progressbar=partial(tqdm, desc='coadd') if verbose else False, ) From 7f7b646ad3d15ee20b70fb8eb4bd61ea830ce773 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Sun, 3 Sep 2023 16:45:55 -0400 Subject: [PATCH 30/36] bugfix in weight handling --- spectral_cube/cube_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 3b9e6b79..db1715fc 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -1055,6 +1055,8 @@ def update(self, n=1): for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, weightcubes), delay=5, desc='Subweight')] + wthdus = [(cube._get_filled_data(), cube.wcs) + for cube in std_tqdm(sweightcubes, delay=5, desc='WeightData')] # reproject_and_coadd requires the actual arrays, so this is the convolution step @@ -1082,7 +1084,7 @@ def update(self, n=1): output_array=output_array[ii:ii+1,:,:], output_footprint=output_footprint[ii:ii+1,:,:], reproject_function=reproject_interp, - input_weights=sweightcubes[ii:ii+1].hdu if weightcubes is not None else None, + input_weights=wthdus, progressbar=partial(tqdm, desc='coadd') if verbose else False, ) From 8dc64ea801c84c076f6e3b56fd63d87d88f9c9dd Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Sun, 3 Sep 2023 17:20:15 -0400 Subject: [PATCH 31/36] bugfix in weight handling --- spectral_cube/cube_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index db1715fc..214707a1 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -1055,7 +1055,7 @@ def update(self, n=1): for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, weightcubes), delay=5, desc='Subweight')] - wthdus = [(cube._get_filled_data(), cube.wcs) + wthdus = [cube.hdu for cube in std_tqdm(sweightcubes, delay=5, desc='WeightData')] From 737500a8237b5b1baf71173260f2dc1dae0ef124 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 4 Sep 2023 14:32:08 -0400 Subject: [PATCH 32/36] when operating on lists of cubes, some may be out of range, and they should be semiautomatically excluded --- spectral_cube/cube_utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 214707a1..afa3f848 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -1049,12 +1049,19 @@ def update(self, n=1): .rechunk()) for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes), delay=5, desc='Subcubes')] + # only keep cubes that are in range; the rest get excluded + keep = [cube.shape[0] > 1 for cube in scubes] + if sum(keep) < len(keep): + log.warn(f"Dropping {len(keep)-sum(keep)} cubes out of {len(keep)} because they're out of range") + scubes = [cube for cube, kp in zip(scubes, keep) if kp] if weightcubes is not None: sweightcubes = [cube[ch1:ch2, slices[1], slices[2]] - for (ch1, ch2), slices, cube - in std_tqdm(zip(chans, mincube_slices, weightcubes), - delay=5, desc='Subweight')] + for (ch1, ch2), slices, cube, kp + in std_tqdm(zip(chans, mincube_slices, weightcubes, kp), + delay=5, desc='Subweight') + if kp + ] wthdus = [cube.hdu for cube in std_tqdm(sweightcubes, delay=5, desc='WeightData')] From 5a73e87484e109af7c83fbde03c3bf3d133fbda3 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 5 Sep 2023 09:32:53 -0400 Subject: [PATCH 33/36] need more checks for cubes to be in range --- spectral_cube/cube_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index afa3f848..c92fd051 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -1050,7 +1050,10 @@ def update(self, n=1): for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes), delay=5, desc='Subcubes')] # only keep cubes that are in range; the rest get excluded - keep = [cube.shape[0] > 1 for cube in scubes] + keep = [(cube.shape[0] > 1) and + (cube.spectral_axis.min() < channel) and + (cube.spectral_axis.max() > channel) + for cube in scubes] if sum(keep) < len(keep): log.warn(f"Dropping {len(keep)-sum(keep)} cubes out of {len(keep)} because they're out of range") scubes = [cube for cube, kp in zip(scubes, keep) if kp] @@ -1058,7 +1061,7 @@ def update(self, n=1): if weightcubes is not None: sweightcubes = [cube[ch1:ch2, slices[1], slices[2]] for (ch1, ch2), slices, cube, kp - in std_tqdm(zip(chans, mincube_slices, weightcubes, kp), + in std_tqdm(zip(chans, mincube_slices, weightcubes, keep), delay=5, desc='Subweight') if kp ] From 0fde3237881d290d4daf6b91b23f86bdb3b969c2 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 5 Sep 2023 13:58:45 -0400 Subject: [PATCH 34/36] have to account for more corner cases for missing data --- spectral_cube/cube_utils.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index c92fd051..7166385b 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -15,6 +15,7 @@ from astropy.wcs import WCS from . import wcs_utils from .utils import FITSWarning, AstropyUserWarning, WCSCelestialError +from .masks import BooleanArrayMask from astropy import log from astropy.io import fits from astropy.wcs.utils import is_proj_plane_distorted @@ -1044,21 +1045,32 @@ def update(self, n=1): with warnings.catch_warnings(): warnings.simplefilter('ignore') # seriously NO WARNINGS. + # exclude any cubes with invalid spatial slices (could happen if whole slices are masked out) + keep1 = [all(x > 1 for x in cube[ch1:ch2, slices[1], slices[2]].shape) + for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes))] + if sum(keep1) < len(keep1): + log.warn(f"Dropping {len(keep1)-sum(keep1)} cubes out of {len(keep1)} because they have invalid (empty) slices") + scubes = [(cube[ch1:ch2, slices[1], slices[2]] .convolve_to(commonbeam) .rechunk()) - for (ch1, ch2), slices, cube in std_tqdm(zip(chans, mincube_slices, cubes), - delay=5, desc='Subcubes')] - # only keep cubes that are in range; the rest get excluded - keep = [(cube.shape[0] > 1) and + for (ch1, ch2), slices, cube, kp in std_tqdm(zip(chans, mincube_slices, cubes, keep1), + delay=5, desc='Subcubes') + if kp + ] + + # only keep2 cubes that are in range; the rest get excluded + keep2 = [all(sh > 1 for sh in cube.shape) and (cube.spectral_axis.min() < channel) and (cube.spectral_axis.max() > channel) for cube in scubes] - if sum(keep) < len(keep): - log.warn(f"Dropping {len(keep)-sum(keep)} cubes out of {len(keep)} because they're out of range") - scubes = [cube for cube, kp in zip(scubes, keep) if kp] + if sum(keep2) < len(keep2): + log.warn(f"Dropping {len(keep2)-sum(keep2)} cubes out of {len(keep2)} because they're out of range") + scubes = [cube for cube, kp in zip(scubes, keep2) if kp] if weightcubes is not None: + # merge the two 'keep' arrays + keep = np.array(keep1) & np.array(keep2) sweightcubes = [cube[ch1:ch2, slices[1], slices[2]] for (ch1, ch2), slices, cube, kp in std_tqdm(zip(chans, mincube_slices, weightcubes, keep), From 15ad95554e0b462e8ad2909e1376fe1112539f5d Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Tue, 5 Sep 2023 20:48:05 -0400 Subject: [PATCH 35/36] fix more bugs with keep --- spectral_cube/cube_utils.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 7166385b..96c64198 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -1054,23 +1054,25 @@ def update(self, n=1): scubes = [(cube[ch1:ch2, slices[1], slices[2]] .convolve_to(commonbeam) .rechunk()) + if kp + else None # placeholder, will drop this below but need to retain list shape for (ch1, ch2), slices, cube, kp in std_tqdm(zip(chans, mincube_slices, cubes, keep1), delay=5, desc='Subcubes') - if kp ] # only keep2 cubes that are in range; the rest get excluded - keep2 = [all(sh > 1 for sh in cube.shape) and - (cube.spectral_axis.min() < channel) and - (cube.spectral_axis.max() > channel) - for cube in scubes] - if sum(keep2) < len(keep2): - log.warn(f"Dropping {len(keep2)-sum(keep2)} cubes out of {len(keep2)} because they're out of range") - scubes = [cube for cube, kp in zip(scubes, keep2) if kp] + keep2 = [(cube is not None) and + all(sh > 1 for sh in cube.shape) and + (cube.spectral_axis.min() < channel) and + (cube.spectral_axis.max() > channel) + for cube in scubes] + # merge the two 'keep' arrays + keep = np.array(keep1) & np.array(keep2) + if sum(keep) < len(keep): + log.warn(f"Dropping {len(keep2)-sum(keep2)} cubes out of {len(keep)} because they're out of range") + scubes = [cube for cube, kp in zip(scubes, keep) if kp] if weightcubes is not None: - # merge the two 'keep' arrays - keep = np.array(keep1) & np.array(keep2) sweightcubes = [cube[ch1:ch2, slices[1], slices[2]] for (ch1, ch2), slices, cube, kp in std_tqdm(zip(chans, mincube_slices, weightcubes, keep), From 10e3ed3e86df7cda6a5b199242ce399d2e145beb Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Fri, 8 Sep 2023 18:49:22 -0400 Subject: [PATCH 36/36] refactor neighboring channel chooser --- spectral_cube/cube_utils.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/spectral_cube/cube_utils.py b/spectral_cube/cube_utils.py index 96c64198..6af578b3 100644 --- a/spectral_cube/cube_utils.py +++ b/spectral_cube/cube_utils.py @@ -833,6 +833,7 @@ def mosaic_cubes(cubes, spectral_block_size=100, combine_header_kwargs={}, output_file=None, method='cube', verbose=True, + fail_if_cube_dropped=False, **kwargs): ''' This function reprojects cubes onto a common grid and combines them to a single field. @@ -1036,8 +1037,16 @@ def update(self, n=1): # this is very verbose but quite simple & cheap # going to spectral_slab(channel-dx, channel+dx) gives 3-pixel cubes most often, # which results in a 50% overhead in smoothing, etc. - chans = [(cube.closest_spectral_channel(channel) if cube.spectral_axis[cube.closest_spectral_channel(channel)] < channel else cube.closest_spectral_channel(channel)+1, - cube.closest_spectral_channel(channel) if cube.spectral_axis[cube.closest_spectral_channel(channel)] > channel else cube.closest_spectral_channel(channel)-1) + def two_closest_channels(cube, channel): + dist = np.abs(cube.spectral_axis.to(channel.unit) - channel) + closest = np.argmin(dist) + dist[closest] = np.inf + next_closest = np.argmin(dist) + if closest < next_closest: + return (closest, next_closest) + else: + return (next_closest, closest) + chans = [two_closest_channels(cube, channel) for cube in std_tqdm(cubes, delay=5, desc='ChanSel:')] # reversed spectral axes still break things # and we want two channels width, not one @@ -1070,6 +1079,8 @@ def update(self, n=1): keep = np.array(keep1) & np.array(keep2) if sum(keep) < len(keep): log.warn(f"Dropping {len(keep2)-sum(keep2)} cubes out of {len(keep)} because they're out of range") + if fail_if_cube_dropped: + raise ValueError(f"There were {len(keep)-sum(keep)} dropped cubes and fail_if_cube_dropped was set") scubes = [cube for cube, kp in zip(scubes, keep) if kp] if weightcubes is not None: @@ -1121,4 +1132,4 @@ def update(self, n=1): hdu.flush() hdu.close() - return cube + return cube \ No newline at end of file