From 3379aba21650b458065356e35f5a99f53c65f801 Mon Sep 17 00:00:00 2001 From: Naim Goksel Karacayli Date: Fri, 17 Mar 2023 11:00:31 -0400 Subject: [PATCH 1/3] Spectrum.write: use_ivar_sm option added to save weights using smooth ivar --- py/qsonic/spectrum.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/py/qsonic/spectrum.py b/py/qsonic/spectrum.py index a6856b6..0cd1caa 100644 --- a/py/qsonic/spectrum.py +++ b/py/qsonic/spectrum.py @@ -398,19 +398,24 @@ def mean_snr(self): npix += armpix return snr / npix - def write(self, fts_file, varlss_interp): + def write(self, fts_file, varlss_interp, use_ivar_sm=False): """Writes each arm to FITS file separately. Writes 'LAMBDA', 'DELTA', 'IVAR', 'WEIGHT', 'CONT' columns and 'RESOMAT' column if resolution matrix is present to extention name - 'targetid-arm'. FITS file must be initialized before. Each arm has its - own `MEANSNR`. `weights` in the file are **not** smoothed. + ``targetid-arm``. FITS file must be initialized before. + Each arm has its own `MEANSNR`. "WEIGHT" in the file do not use + smoothed inverse variance by default, which can be changed by + ``use_ivar_sm=True``. Arguments --------- fts_file: FITS file + The file handler, not filename. varlss_interp: Fast1DInterpolator or any other interpolator. LSS variance interpolator. + use_ivar_sm: bool, default: False + Use :attr:`forestivar_sm` in weights instead. """ hdr_dict = { 'LOS_ID': self.targetid, @@ -423,6 +428,7 @@ def write(self, fts_file, varlss_interp): 'MEANSNR': 0., 'RSNR': self.rsnr, 'DELTA_LAMBDA': self.dwave, + 'SMWEIGHT': use_ivar_sm, } for arm, wave_arm in self.forestwave.items(): @@ -439,7 +445,9 @@ def write(self, fts_file, varlss_interp): delta = self.forestflux[arm] / cont_est - 1 ivar = self.forestivar[arm] * cont_est**2 var_lss = varlss_interp(wave_arm) - weight = ivar / (1 + ivar * var_lss) + ivar_w = ( + self.forestivar_sm[arm] * cont_est**2 if use_ivar_sm else ivar) + weight = ivar_w / (1 + ivar_w * var_lss) cols = [wave_arm, delta, ivar, weight, cont_est] if self.forestreso: From 51ad90eb746dff0bbbfad53084c3d649ca1b85b7 Mon Sep 17 00:00:00 2001 From: Naim Goksel Karacayli Date: Fri, 17 Mar 2023 11:09:54 -0400 Subject: [PATCH 2/3] io.add_io_parser: splits arguments into input and output groups separately, and --save-smooth-weights option to save smooth weights. io.save_deltas: use_ivar_sm option added. --- py/qsonic/io.py | 51 ++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/py/qsonic/io.py b/py/qsonic/io.py index f502576..a893947 100644 --- a/py/qsonic/io.py +++ b/py/qsonic/io.py @@ -27,46 +27,49 @@ def add_io_parser(parser=None): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) - iogroup = parser.add_argument_group( - 'Input/output parameters and selections') - iogroup.add_argument( + ingroup = parser.add_argument_group('Input options') + ingroup.add_argument( "--input-dir", '-i', required=True, help="Input directory to healpix") - iogroup.add_argument( + ingroup.add_argument( "--catalog", required=True, help="Catalog filename") - iogroup.add_argument( - "--outdir", '-o', - help="Output directory to save deltas.") - iogroup.add_argument( + ingroup.add_argument( "--mock-analysis", action="store_true", help="Input folder is mock. Uses nside=16") - iogroup.add_argument( + ingroup.add_argument( "--keep-surveys", nargs='+', default=['sv3', 'main'], help="Surveys to keep.") - iogroup.add_argument( + ingroup.add_argument( "--coadd-arms", action="store_true", help="Coadds arms when saving.") - iogroup.add_argument( + ingroup.add_argument( "--skip-resomat", action="store_true", help="Skip reading resolution matrix for 3D.") - iogroup.add_argument( + ingroup.add_argument( "--arms", default=['B', 'R'], choices=['B', 'R', 'Z'], nargs='+', help="Arms to read.") - iogroup.add_argument( + ingroup.add_argument( "--min-rsnr", type=float, default=0., help="Minium SNR above Lya.") - iogroup.add_argument( + ingroup.add_argument( "--skip", type=_float_range(0, 1), default=0., help="Skip short spectra lower than given ratio.") - iogroup.add_argument( - "--save-by-hpx", action="store_true", - help="Save by healpix. If not, saves by MPI rank.") - iogroup.add_argument( + ingroup.add_argument( "--keep-nonforest-pixels", action="store_true", help="Keeps non forest wavelengths. Memory intensive!") + outgroup = parser.add_argument_group('Output options') + outgroup.add_argument( + "--outdir", '-o', + help="Output directory to save deltas.") + outgroup.add_argument( + "--save-smooth-weights", action="store_true", + help="Save smoothed weights instead.") + outgroup.add_argument( + "--save-by-hpx", action="store_true", + help="Save by healpix. If not, saves by MPI rank.") return parser @@ -141,7 +144,9 @@ def read_spectra_onehealpix( def save_deltas( - spectra_list, outdir, varlss_interp, save_by_hpx=False, mpi_rank=None): + spectra_list, outdir, varlss_interp, + save_by_hpx=False, mpi_rank=None, use_ivar_sm=False +): """ Saves given list of spectra as deltas. NO coaddition of arms. Each arm is saved separately. Only valid spectra are saved. @@ -153,10 +158,12 @@ def save_deltas( Output directory. Does not save if empty of None varlss_interp: Interpolator Interpolator for LSS variance - save_by_hpx: bool + save_by_hpx: bool, default: False Saves by healpix if True. Has priority over mpi_rank - mpi_rank: int + mpi_rank: int, default: None Rank of the MPI process. Save by `mpi_rank` if passed. + use_ivar_sm: bool, default: False + Use :attr:`Spectrum.forestivar_sm` in weights instead. Raises --------- @@ -188,7 +195,7 @@ def save_deltas( f"{outdir}/delta-{healpix}.fits", 'rw', clobber=True) for spec in qsonic.spectrum.valid_spectra(hp_specs): - spec.write(results, varlss_interp) + spec.write(results, varlss_interp, use_ivar_sm) results.close() From a247163b2133bc0557bb3e415236215abc07e85f Mon Sep 17 00:00:00 2001 From: Naim Goksel Karacayli Date: Fri, 17 Mar 2023 11:15:56 -0400 Subject: [PATCH 3/3] io.add_io_parser: move coadd-arms to output group, move min-rsnr, skip and keep-nonforest-pixels to qsonic_fit.get_parser analysis_group. --- py/qsonic/io.py | 16 +++------------- py/qsonic/scripts/qsonic_fit.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/py/qsonic/io.py b/py/qsonic/io.py index a893947..fc2ad60 100644 --- a/py/qsonic/io.py +++ b/py/qsonic/io.py @@ -40,30 +40,20 @@ def add_io_parser(parser=None): ingroup.add_argument( "--keep-surveys", nargs='+', default=['sv3', 'main'], help="Surveys to keep.") - ingroup.add_argument( - "--coadd-arms", action="store_true", - help="Coadds arms when saving.") - ingroup.add_argument( "--skip-resomat", action="store_true", help="Skip reading resolution matrix for 3D.") ingroup.add_argument( "--arms", default=['B', 'R'], choices=['B', 'R', 'Z'], nargs='+', help="Arms to read.") - ingroup.add_argument( - "--min-rsnr", type=float, default=0., - help="Minium SNR above Lya.") - ingroup.add_argument( - "--skip", type=_float_range(0, 1), default=0., - help="Skip short spectra lower than given ratio.") - ingroup.add_argument( - "--keep-nonforest-pixels", action="store_true", - help="Keeps non forest wavelengths. Memory intensive!") outgroup = parser.add_argument_group('Output options') outgroup.add_argument( "--outdir", '-o', help="Output directory to save deltas.") + outgroup.add_argument( + "--coadd-arms", action="store_true", + help="Coadds arms when saving.") outgroup.add_argument( "--save-smooth-weights", action="store_true", help="Save smoothed weights instead.") diff --git a/py/qsonic/scripts/qsonic_fit.py b/py/qsonic/scripts/qsonic_fit.py index f4d0135..59a836c 100644 --- a/py/qsonic/scripts/qsonic_fit.py +++ b/py/qsonic/scripts/qsonic_fit.py @@ -32,6 +32,18 @@ def get_parser(add_help=True): formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser = qsonic.io.add_io_parser(parser) + + analysis_group = parser.add_argument_group('Analysis options') + analysis_group.add_argument( + "--min-rsnr", type=float, default=0., + help="Minium SNR above Lya.") + analysis_group.add_argument( + "--skip", type=qsonic.io._float_range(0, 1), default=0., + help="Skip short spectra lower than given ratio.") + analysis_group.add_argument( + "--keep-nonforest-pixels", action="store_true", + help="Keeps non forest wavelengths. Memory intensive!") + parser = qsonic.spectrum.add_wave_region_parser(parser) parser = qsonic.masks.add_mask_parser(parser) parser = add_picca_continuum_parser(parser)