From 61de922b481a21a263d42d8e72cb62f6f7f104a5 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Tue, 12 Sep 2023 17:12:42 -0700 Subject: [PATCH 1/8] fix zcat cumulative LASTNIGHT stacking --- bin/desi_zcatalog | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/desi_zcatalog b/bin/desi_zcatalog index 15d34286b..5fa5df1a3 100755 --- a/bin/desi_zcatalog +++ b/bin/desi_zcatalog @@ -293,8 +293,9 @@ for ifile, rrfile in enumerate(redrockfiles): data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), index=icol, name='NIGHT') elif args.group == 'cumulative': - data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), - index=icol, name='LASTNIGHT') + if 'LASTNIGHT' not in data.colnames: + data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), + index=icol, name='LASTNIGHT') elif args.group == 'healpix': data.add_column(np.full(nrows, hdr['HPXPIXEL'], dtype=np.int32), index=icol, name='HEALPIX') From 82c7cec7bc0442897bfb568470cc1f555de670d1 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Tue, 12 Sep 2023 17:06:00 -0700 Subject: [PATCH 2/8] WIP desi_zcatalog refactor --- bin/desi_zcatalog | 473 ++++++++++++++++++++++++++-------------------- 1 file changed, 267 insertions(+), 206 deletions(-) diff --git a/bin/desi_zcatalog b/bin/desi_zcatalog index 5fa5df1a3..a3155c885 100755 --- a/bin/desi_zcatalog +++ b/bin/desi_zcatalog @@ -18,6 +18,7 @@ from __future__ import absolute_import, division, print_function import sys, os, glob import argparse import importlib.resources +import multiprocessing as mp import numpy as np from numpy.lib.recfunctions import append_fields @@ -136,101 +137,35 @@ def load_sv1_ivar_w12(hpix, targetids): return targets +def _wrap_read_redrock(optdict): + return read_redrock(**optdict) -#-------------------------------------------------------------------------- +def read_redrock(rrfile, group=None, recoadd_fibermap=False, minimal=False, pertile=False, counter=None): + """ + TODO: document + """ + log = get_logger() + if counter is not None: + i, n = counter + log.info(f'Reading {i}/{n} {rrfile}') + else: + log.info(f'Reading {rrfile}') -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument("-i", "--indir", type=str, - help="input directory") -parser.add_argument("-o", "--outfile",type=str, - help="output file") -parser.add_argument("--minimal", action='store_true', - help="only include minimal output columns") -parser.add_argument("-t", "--tiles", type=str, - help="ascii file with tileids to include (one per line)") -parser.add_argument("--prefix", type=str, default='redrock', - help="prefix of redrock files (older versions used 'zbest' " - "instead of 'redrock'") -parser.add_argument("-g", "--group", type=str, - help="Add columns specific to this spectral grouping " - "e.g. pernight adds NIGHT column from input header keyword") -parser.add_argument("--header", type=str, nargs="*", - help="KEYWORD=VALUE entries to add to the output header") -parser.add_argument('--patch-missing-ivar-w12', action='store_true', - help="Use target files to patch missing FLUX_IVAR_W1/W2 values") -parser.add_argument('--recoadd-fibermap', action='store_true', - help="Re-coadd FIBERMAP from spectra files") -parser.add_argument('--ztile', action='store_true', - help="Used with --recoadd-fibermap, this is a tile-based recoadd " - "not a healpix-based recoadd") -parser.add_argument('--add-units', action='store_true', - help="Add units to output catalog from desidatamodel " - "column descriptions") - -# parser.add_argument("--match", type=str, nargs="*", -# help="match other tables (targets,truth...)") - -args = parser.parse_args() - -log=get_logger() - -if args.indir is None: - log.error('--indir directory required') - sys.exit(1) - -if args.outfile is None: - args.outfile = io.findfile('zcatalog') - -#- If adding units, check dependencies before doing a lot of work -if args.add_units: - try: - import desidatamodel - except ImportError: - log.critical('Unable to import desidatamodel, required to add units (try "module load desidatamodel" first)') - sys.exit(1) - -#- Get redrock*.fits files in subdirs, excluding e.g. redrock*.log - -log.info(f'Looking for redrock files in subdirectories of {args.indir}') -if args.tiles is not None: - tiles = np.atleast_1d(np.loadtxt(args.tiles, dtype=int)) - ntiles = len(tiles) - log.info(f'Filtering to {ntiles} tiles from {args.tiles}') - redrockfiles = list() - for tileid in tiles: - tmp = sorted(io.iterfiles(f'{args.indir}/{tileid}', prefix=args.prefix, suffix='.fits')) - if len(tmp) > 0: - redrockfiles.extend(tmp) - else: - log.error(f'no redrock files found in {args.indir}/{tileid}') -else: - redrockfiles = sorted(io.iterfiles(args.indir, prefix=args.prefix, suffix='.fits')) - -nfiles = len(redrockfiles) -if nfiles == 0: - msg = f'No redrock files found in {args.indir}' - log.critical(msg) - raise ValueError(msg) - -zcatdata = list() -exp_fibermaps = list() -for ifile, rrfile in enumerate(redrockfiles): - log.info(f'Reading {ifile+1}/{nfiles} {rrfile}') with fitsio.FITS(rrfile) as fx: hdr = fx[0].read_header() - if args.group is not None and 'SPGRP' in hdr and \ - hdr['SPGRP'] != args.group: - log.warning("Skipping {} with SPGRP {} != args.group {}".format( - rrfile, hdr['SPGRP'], args.group)) - continue + if group is not None and 'SPGRP' in hdr and \ + hdr['SPGRP'] != group: + log.warning("Skipping {} with SPGRP {} != group {}".format( + rrfile, hdr['SPGRP'], group)) + return None redshifts = fx['REDSHIFTS'].read() - if args.recoadd_fibermap: + if recoadd_fibermap: spectra_filename = checkgzip(replace_prefix(rrfile, 'redrock', 'spectra')) log.info('Recoadding fibermap from %s', os.path.basename(spectra_filename)) fibermap_orig = read_table(spectra_filename) - fibermap, expfibermap = coadd_fibermap(fibermap_orig, onetile=args.ztile) + fibermap, expfibermap = coadd_fibermap(fibermap_orig, onetile=pertile) else: fibermap = fx['FIBERMAP'].read() expfibermap = fx['EXP_FIBERMAP'].read() @@ -239,20 +174,13 @@ for ifile, rrfile in enumerate(redrockfiles): assert np.all(redshifts['TARGETID'] == fibermap['TARGETID']) assert np.all(redshifts['TARGETID'] == tsnr2['TARGETID']) - if args.minimal: + if minimal: fmcols = ['TARGET_RA', 'TARGET_DEC', 'FLUX_G', 'FLUX_R', 'FLUX_Z'] for colname in fibermap.dtype.names: if colname.endswith('_TARGET') and colname != 'FA_TARGET': fmcols.append(colname) - if args.prefix == 'zbest': - fibermap_=Table(fibermap[fmcols]) - fibermap_.rename_column('TARGET_RA','RA') - fibermap_.rename_column('TARGET_DEC','DEC') - fibermap_.remove_columns(['DESI_TARGET','BGS_TARGET','MWS_TARGET','SCND_TARGET']) - data = hstack( [Table(redshifts), fibermap_] ) - else: - data = hstack( [Table(redshifts), Table(fibermap[fmcols])] ) + data = hstack( [Table(redshifts), Table(fibermap[fmcols])] ) else: fmcols = list(fibermap.dtype.names) @@ -273,7 +201,7 @@ for ifile, rrfile in enumerate(redrockfiles): #- Put these columns right after TARGETID nrows = len(data) icol = 1 - if args.group in ('perexp', 'pernight', 'cumulative'): + if group in ('perexp', 'pernight', 'cumulative'): if 'TILEID' not in data.colnames: data.add_column(np.full(nrows, hdr['TILEID'], dtype=np.int32), index=icol, name='TILEID') @@ -283,13 +211,13 @@ for ifile, rrfile in enumerate(redrockfiles): index=icol, name='PETAL_LOC') icol += 1 - if args.group == 'perexp': + if group == 'perexp': data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), index=icol, name='NIGHT') icol += 1 data.add_column(np.full(nrows, hdr['EXPID'], dtype=np.int32), index=icol, name='EXPID') - elif args.group == 'pernight': + elif group == 'pernight': data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), index=icol, name='NIGHT') elif args.group == 'cumulative': @@ -319,114 +247,247 @@ for ifile, rrfile in enumerate(redrockfiles): else: log.warning(f'SPGRPVAL keyword missing from {rrfile}') - zcatdata.append(data) - - if expfibermap is not None: - exp_fibermaps.append(expfibermap) - - -log.info('Stacking zcat') -zcat = vstack(zcatdata) -if exp_fibermaps: - log.info('Stacking exposure fibermaps') - expfm = np.hstack(exp_fibermaps) -else: - expfm = None - -#- if TARGETIDs appear more than once, which one is best within this catalog? -if 'TSNR2_LRG' in zcat.colnames and 'ZWARN' in zcat.colnames: - log.info('Finding best spectrum for each target') - nspec, primary = find_primary_spectra(zcat) - zcat['ZCAT_NSPEC'] = nspec.astype(np.int16) - zcat['ZCAT_PRIMARY'] = primary -else: - log.info('Missing TSNR2_LRG or ZWARN; not adding ZCAT_PRIMARY/_NSPEC') - -if args.patch_missing_ivar_w12: - from desimodel.footprint import radec2pix - missing = (zcat['FLUX_IVAR_W1'] < 0) | (zcat['FLUX_IVAR_W2'] < 0) - missing &= zcat['OBJTYPE'] == 'TGT' - missing &= zcat['TARGETID'] > 0 - - if not np.any(missing): - log.info('No targets missing FLUX_IVAR_W1/W2 to patch') + return data, expfibermap + +#-------------------------------------------------------------------------- + +""" +# stack specprod healpix, optionally filtering by SURVEY and PROGRAM +desi_zcatalog --group healpix --survey main --program dark -o zpix-main-dark.fits + +# stack specprod tiles, optionally filtering by SURVEY and PROGRAM +desi_zcatalog --group cumulative --survey main --program dark -o ztile-main-dark.fits + +# load whatever it can find in a directory +desi_zcatalog --indir testprod/tiles/cumulative/ -o zcat.fits +desi_zcatalog --indir testprod/tiles/cumulative/ --tiles tiles.txt -o zcat-subset.fits + +""" + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-i", "--indir", type=str, + help="input directory") + parser.add_argument("-o", "--outfile",type=str, + help="output file") + parser.add_argument("--minimal", action='store_true', + help="only include minimal output columns") + parser.add_argument("-t", "--tiles", type=str, + help="ascii file with tileids to include (one per line)") + + parser.add_argument("--survey", type=str, + help="DESI survey, e.g. sv1, sv3, main") + parser.add_argument("--program", type=str, + help="DESI program, e.g bright, dark") + + parser.add_argument("-g", "--group", type=str, + help="Add columns specific to this spectral grouping " + "e.g. pernight adds NIGHT column from input header keyword") + parser.add_argument("--header", type=str, nargs="*", + help="KEYWORD=VALUE entries to add to the output header") + parser.add_argument('--patch-missing-ivar-w12', action='store_true', + help="Use target files to patch missing FLUX_IVAR_W1/W2 values") + parser.add_argument('--recoadd-fibermap', action='store_true', + help="Re-coadd FIBERMAP from spectra files") + parser.add_argument('--add-units', action='store_true', + help="Add units to output catalog from desidatamodel " + "column descriptions") + parser.add_argument('--nproc', type=int, default=1, + help="Number of multiprocessing processes to use") + + # parser.add_argument("--match", type=str, nargs="*", + # help="match other tables (targets,truth...)") + + args = parser.parse_args() + + log=get_logger() + + if args.outfile is None: + args.outfile = io.findfile('zcatalog') + + #- If adding units, check dependencies before doing a lot of work + if args.add_units: + try: + import desidatamodel + except ImportError: + log.critical('Unable to import desidatamodel, required to add units (try "module load desidatamodel" first)') + sys.exit(1) + + if args.group == 'healpix': + pertile = False + survey = args.survey if args.survey is not None else "*" + program = args.program if args.program is not None else "*" + hpixdir = args.indir if args.indir is not None else os.path.join(io.specprod_root(), 'healpix') + + #- specprod/healpix/SURVEY/PROGRAM/HPIXGROUP/HPIX/redrock*.fits + globstr = os.path.join(hpixdir, survey, program, '*', '*', 'redrock*.fits') + log.info(f'Looking for healpix redrock files in {globstr}') + redrockfiles = sorted(glob.glob(globstr)) else: - #- Load targets from sv1 targeting files - ra = zcat['TARGET_RA'] - dec = zcat['TARGET_DEC'] - nside = 8 #- use for sv1 targeting - hpix8 = radec2pix(nside, ra, dec) - for hpix in np.unique(hpix8[missing]): - hpixmiss = (hpix == hpix8) & missing - targets = load_sv1_ivar_w12(hpix, zcat['TARGETID'][hpixmiss]) - - #- create dict[TARGETID] -> row number - targetid2idx = dict(zip(targets['TARGETID'], - np.arange(len(targets)))) - - #- patch missing values, if they are in the targets file - for i in np.where(hpixmiss)[0]: - tid = zcat['TARGETID'][i] - try: - j = targetid2idx[ tid ] - zcat['FLUX_IVAR_W1'][i] = targets['FLUX_IVAR_W1'][j] - zcat['FLUX_IVAR_W2'][i] = targets['FLUX_IVAR_W2'][j] - except KeyError: - log.warning(f'TARGETID {tid} (row {i}) not found in sv1 targets') - -#- we're done adding columns, convert to numpy array for fitsio -zcat = np.array(zcat) - -#- untested with new formats, so commenting out for now -# if args.match: -# for filename in args.match : -# log.info("matching {}".format(filename)) -# zcat = match(zcat,fitsio.read(filename)) - -#- Inherit header from first input, but remove keywords that don't apply -#- across multiple files -header = fitsio.read_header(redrockfiles[0], 0) -for key in ['SPGRPVAL', 'TILEID', 'SPECTRO', 'PETAL', 'NIGHT', 'EXPID', 'HPXPIXEL', - 'NAXIS', 'BITPIX', 'SIMPLE', 'EXTEND']: - if key in header: - header.delete(key) - -#- Intercept previous incorrect boolean special cases -if 'HPXNEST' in header: - if header['HPXNEST'] == 'True': - log.info("Correcting header HPXNEST='True' string to boolean True") - header['HPXNEST'] = True - elif header['HPXNEST'] == 'False': - # False is not expected for DESI, but cover it for completeness - log.info("Correcting header HPXNEST='False' string to boolean False") - header['HPXNEST'] = False - -#- Add extra keywords if requested -if args.header is not None: - for keyval in args.header: - key, value = parse_keyval(keyval) - header[key] = value - -#- Add units if requested -if args.add_units: - datamodeldir = str(importlib.resources.files('desidatamodel')) - unitsfile = os.path.join(datamodeldir, 'data', 'column_descriptions.csv') - log.info(f'Adding units from {unitsfile}') - units, comments = load_csv_units(unitsfile) -else: - units = dict() - comments = dict() - -log.info(f'Writing {args.outfile}') -tmpfile = get_tempfilename(args.outfile) - -write_bintable(tmpfile, zcat, header=header, extname='ZCATALOG', - units=units, clobber=True) - -if not args.minimal and expfm is not None: - write_bintable(tmpfile, expfm, extname='EXP_FIBERMAP', units=units) - -os.rename(tmpfile, args.outfile) - -log.info("Successfully wrote {}".format(args.outfile)) + pertile = True + tilefile = args.tiles if args.tiles is not None else io.findfile('tiles') + indir = args.indir if args.indir is not None else os.path.join(io.specprod_root(), 'tiles', args.group) + + log.info(f'Loading tiles from {tilefile}') + tiles = Table.read(tilefile) + if args.survey is not None: + keep = tiles['SURVEY'] == args.survey + tiles = tiles[keep] + if len(tiles) == 0: + log.critical(f'No tiles kept after filtering by SURVEY={args.survey}') + sys.exit(1) + + if args.program is not None: + keep = tiles['PROGRAM'] == args.program + tile = tiles[keep] + if len(tiles) == 0: + log.critical(f'No tiles kept after filtering by PROGRAM={args.program}') + sys.exit(1) + + tileids = tiles['TILEID'] + + redrockfiles = list() + for tileid in tileids: + tmp = sorted(io.iterfiles(f'{indir}/{tileid}', prefix='redrock', suffix='.fits')) + if len(tmp) > 0: + redrockfiles.extend(tmp) + else: + log.error(f'no redrock files found in {args.indir}/{tileid}') + + + nfiles = len(redrockfiles) + if nfiles == 0: + msg = f'No redrock files found in {args.indir}' + log.critical(msg) + raise ValueError(msg) + log.info(f'Reading {nfiles} redrock files') + + read_args = list() + for ifile, rrfile in enumerate(redrockfiles): + read_args.append(dict(rrfile=rrfile, group=args.group, pertile=pertile, + recoadd_fibermap=args.recoadd_fibermap, minimal=args.minimal, + counter=(ifile+1, nfiles))) + + if args.nproc>1: + from multiprocessing import Pool + with Pool(args.nproc) as pool: + results = pool.map(_wrap_read_redrock, read_args) + else: + results = [_wrap_read_redrock(a) for a in read_args] + + zcatdata = list() + exp_fibermaps = list() + for data, expfibermap in results: + if data is not None: + zcatdata.append(data) + + if expfibermap is not None: + exp_fibermaps.append(expfibermap) + + log.info('Stacking zcat') + zcat = vstack(zcatdata) + if exp_fibermaps: + log.info('Stacking exposure fibermaps') + expfm = np.hstack(exp_fibermaps) + else: + expfm = None + + #- if TARGETIDs appear more than once, which one is best within this catalog? + if 'TSNR2_LRG' in zcat.colnames and 'ZWARN' in zcat.colnames: + log.info('Finding best spectrum for each target') + nspec, primary = find_primary_spectra(zcat) + zcat['ZCAT_NSPEC'] = nspec.astype(np.int16) + zcat['ZCAT_PRIMARY'] = primary + else: + log.info('Missing TSNR2_LRG or ZWARN; not adding ZCAT_PRIMARY/_NSPEC') + + if args.patch_missing_ivar_w12: + from desimodel.footprint import radec2pix + missing = (zcat['FLUX_IVAR_W1'] < 0) | (zcat['FLUX_IVAR_W2'] < 0) + missing &= zcat['OBJTYPE'] == 'TGT' + missing &= zcat['TARGETID'] > 0 + + if not np.any(missing): + log.info('No targets missing FLUX_IVAR_W1/W2 to patch') + else: + #- Load targets from sv1 targeting files + ra = zcat['TARGET_RA'] + dec = zcat['TARGET_DEC'] + nside = 8 #- use for sv1 targeting + hpix8 = radec2pix(nside, ra, dec) + for hpix in np.unique(hpix8[missing]): + hpixmiss = (hpix == hpix8) & missing + targets = load_sv1_ivar_w12(hpix, zcat['TARGETID'][hpixmiss]) + + #- create dict[TARGETID] -> row number + targetid2idx = dict(zip(targets['TARGETID'], + np.arange(len(targets)))) + + #- patch missing values, if they are in the targets file + for i in np.where(hpixmiss)[0]: + tid = zcat['TARGETID'][i] + try: + j = targetid2idx[ tid ] + zcat['FLUX_IVAR_W1'][i] = targets['FLUX_IVAR_W1'][j] + zcat['FLUX_IVAR_W2'][i] = targets['FLUX_IVAR_W2'][j] + except KeyError: + log.warning(f'TARGETID {tid} (row {i}) not found in sv1 targets') + + #- we're done adding columns, convert to numpy array for fitsio + zcat = np.array(zcat) + + #- untested with new formats, so commenting out for now + # if args.match: + # for filename in args.match : + # log.info("matching {}".format(filename)) + # zcat = match(zcat,fitsio.read(filename)) + + #- Inherit header from first input, but remove keywords that don't apply + #- across multiple files + header = fitsio.read_header(redrockfiles[0], 0) + for key in ['SPGRPVAL', 'TILEID', 'SPECTRO', 'PETAL', 'NIGHT', 'EXPID', 'HPXPIXEL', + 'NAXIS', 'BITPIX', 'SIMPLE', 'EXTEND']: + if key in header: + header.delete(key) + + #- Intercept previous incorrect boolean special cases + if 'HPXNEST' in header: + if header['HPXNEST'] == 'True': + log.info("Correcting header HPXNEST='True' string to boolean True") + header['HPXNEST'] = True + elif header['HPXNEST'] == 'False': + # False is not expected for DESI, but cover it for completeness + log.info("Correcting header HPXNEST='False' string to boolean False") + header['HPXNEST'] = False + + #- Add extra keywords if requested + if args.header is not None: + for keyval in args.header: + key, value = parse_keyval(keyval) + header[key] = value + + #- Add units if requested + if args.add_units: + datamodeldir = str(importlib.resources.files('desidatamodel')) + unitsfile = os.path.join(datamodeldir, 'data', 'column_descriptions.csv') + log.info(f'Adding units from {unitsfile}') + units, comments = load_csv_units(unitsfile) + else: + units = dict() + comments = dict() + + log.info(f'Writing {args.outfile}') + tmpfile = get_tempfilename(args.outfile) + + write_bintable(tmpfile, zcat, header=header, extname='ZCATALOG', + units=units, clobber=True) + + if not args.minimal and expfm is not None: + write_bintable(tmpfile, expfm, extname='EXP_FIBERMAP', units=units) + + os.rename(tmpfile, args.outfile) + + log.info("Successfully wrote {}".format(args.outfile)) +if __name__ == '__main__': + main() From e1ae2a517fa193b3cff7a9de2e81089b8143b9e6 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 13 Sep 2023 11:36:51 -0700 Subject: [PATCH 3/8] fix typos; add docs --- bin/desi_zcatalog | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bin/desi_zcatalog b/bin/desi_zcatalog index a3155c885..578e8a475 100755 --- a/bin/desi_zcatalog +++ b/bin/desi_zcatalog @@ -138,11 +138,25 @@ def load_sv1_ivar_w12(hpix, targetids): return targets def _wrap_read_redrock(optdict): + """read_redrock wrapper to expand dictionary of named args for multiprocessing""" return read_redrock(**optdict) def read_redrock(rrfile, group=None, recoadd_fibermap=False, minimal=False, pertile=False, counter=None): """ - TODO: document + Read a Redrock file + + Args: + rrfile (str): full path to redrock filename + + Options: + group (str): add group-specific columns for cumulative, pernight, healpix + readcoadd_fibermap (bool): recoadd fibermap from spectra file in same dir + minimal (bool): only propagate minimal subet of columns + pertile (bool): input Redrock file is single tile (not healpix) + counter (tuple): (i,n) log loading ith file out of n + + Returns (zcat, expfibermap) where zcat is a join of the redrock REDSHIFTS + catalog and the coadded FIBERMAP """ log = get_logger() if counter is not None: @@ -220,7 +234,7 @@ def read_redrock(rrfile, group=None, recoadd_fibermap=False, minimal=False, pert elif group == 'pernight': data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), index=icol, name='NIGHT') - elif args.group == 'cumulative': + elif group == 'cumulative': if 'LASTNIGHT' not in data.colnames: data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), index=icol, name='LASTNIGHT') @@ -339,7 +353,7 @@ def main(): if args.program is not None: keep = tiles['PROGRAM'] == args.program - tile = tiles[keep] + tiles = tiles[keep] if len(tiles) == 0: log.critical(f'No tiles kept after filtering by PROGRAM={args.program}') sys.exit(1) From 5c88026b6c87f1ec21ef1b0e5892ed48de8cc73e Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Wed, 13 Sep 2023 12:07:24 -0700 Subject: [PATCH 4/8] fix --indir usage --- bin/desi_zcatalog | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/bin/desi_zcatalog b/bin/desi_zcatalog index 578e8a475..a308431b9 100755 --- a/bin/desi_zcatalog +++ b/bin/desi_zcatalog @@ -11,6 +11,7 @@ context of this script. Stephen Bailey Lawrence Berkeley National Lab Fall 2015 +substantially updated Fal 2023 """ from __future__ import absolute_import, division, print_function @@ -143,7 +144,7 @@ def _wrap_read_redrock(optdict): def read_redrock(rrfile, group=None, recoadd_fibermap=False, minimal=False, pertile=False, counter=None): """ - Read a Redrock file + Read a Redrock file, combining REDSHIFTS, FIBERMAP, and TSNR2 HDUs Args: rrfile (str): full path to redrock filename @@ -238,7 +239,7 @@ def read_redrock(rrfile, group=None, recoadd_fibermap=False, minimal=False, pert if 'LASTNIGHT' not in data.colnames: data.add_column(np.full(nrows, hdr['NIGHT'], dtype=np.int32), index=icol, name='LASTNIGHT') - elif args.group == 'healpix': + elif group == 'healpix': data.add_column(np.full(nrows, hdr['HPXPIXEL'], dtype=np.int32), index=icol, name='HEALPIX') @@ -265,19 +266,6 @@ def read_redrock(rrfile, group=None, recoadd_fibermap=False, minimal=False, pert #-------------------------------------------------------------------------- -""" -# stack specprod healpix, optionally filtering by SURVEY and PROGRAM -desi_zcatalog --group healpix --survey main --program dark -o zpix-main-dark.fits - -# stack specprod tiles, optionally filtering by SURVEY and PROGRAM -desi_zcatalog --group cumulative --survey main --program dark -o ztile-main-dark.fits - -# load whatever it can find in a directory -desi_zcatalog --indir testprod/tiles/cumulative/ -o zcat.fits -desi_zcatalog --indir testprod/tiles/cumulative/ --tiles tiles.txt -o zcat-subset.fits - -""" - def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-i", "--indir", type=str, @@ -327,20 +315,24 @@ def main(): log.critical('Unable to import desidatamodel, required to add units (try "module load desidatamodel" first)') sys.exit(1) - if args.group == 'healpix': + if args.indir: + indir = args.indir + redrockfiles = sorted(io.iterfiles(f'{indir}', prefix='redrock', suffix='.fits')) + pertile = (args.group != 'healpix') # assume tile-based input unless explicitely healpix + elif args.group == 'healpix': pertile = False survey = args.survey if args.survey is not None else "*" program = args.program if args.program is not None else "*" - hpixdir = args.indir if args.indir is not None else os.path.join(io.specprod_root(), 'healpix') + indir = os.path.join(io.specprod_root(), 'healpix') #- specprod/healpix/SURVEY/PROGRAM/HPIXGROUP/HPIX/redrock*.fits - globstr = os.path.join(hpixdir, survey, program, '*', '*', 'redrock*.fits') + globstr = os.path.join(indir, survey, program, '*', '*', 'redrock*.fits') log.info(f'Looking for healpix redrock files in {globstr}') redrockfiles = sorted(glob.glob(globstr)) else: pertile = True tilefile = args.tiles if args.tiles is not None else io.findfile('tiles') - indir = args.indir if args.indir is not None else os.path.join(io.specprod_root(), 'tiles', args.group) + indir = os.path.join(io.specprod_root(), 'tiles', args.group) log.info(f'Loading tiles from {tilefile}') tiles = Table.read(tilefile) @@ -366,22 +358,24 @@ def main(): if len(tmp) > 0: redrockfiles.extend(tmp) else: - log.error(f'no redrock files found in {args.indir}/{tileid}') + log.error(f'no redrock files found in {indir}/{tileid}') nfiles = len(redrockfiles) if nfiles == 0: - msg = f'No redrock files found in {args.indir}' + msg = f'No redrock files found in {indir}' log.critical(msg) raise ValueError(msg) log.info(f'Reading {nfiles} redrock files') + #- build list of args to support multiprocessing parallelism read_args = list() for ifile, rrfile in enumerate(redrockfiles): read_args.append(dict(rrfile=rrfile, group=args.group, pertile=pertile, recoadd_fibermap=args.recoadd_fibermap, minimal=args.minimal, counter=(ifile+1, nfiles))) + #- Read individual Redrock files if args.nproc>1: from multiprocessing import Pool with Pool(args.nproc) as pool: @@ -389,6 +383,7 @@ def main(): else: results = [_wrap_read_redrock(a) for a in read_args] + #- Stack catalogs zcatdata = list() exp_fibermaps = list() for data, expfibermap in results: @@ -415,6 +410,7 @@ def main(): else: log.info('Missing TSNR2_LRG or ZWARN; not adding ZCAT_PRIMARY/_NSPEC') + #- Used for fuji, should not be needed for later prods if args.patch_missing_ivar_w12: from desimodel.footprint import radec2pix missing = (zcat['FLUX_IVAR_W1'] < 0) | (zcat['FLUX_IVAR_W2'] < 0) From 401348f3132e89d8f68312d44b6530926d35e438 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Thu, 14 Sep 2023 13:29:42 -0700 Subject: [PATCH 5/8] update changes.rst prior to 0.60.0 tag --- doc/changes.rst | 52 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/doc/changes.rst b/doc/changes.rst index 9bd56ed59..8dfe8971e 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -2,12 +2,62 @@ desispec Change Log =================== -0.59.3 (unreleased) +0.60.1 (unreleased) ------------------- +* No changes yet. + +0.60.0 (2023-09-14) +------------------- + +Updates for patching Iron/DR1 redshift catalogs + +* coadd_fibermap handle RA wraparound (PR `#2090`_). +* Update COADD_FIBERSTATUS to bitwise OR when all inputs are bad (PR `#2093`_). +* Coadd bitwise OR of input targeting bits, e.g. DESI_TARGET (PR `#2094`_). +* Add IN_COADD/B/R/Z columns to EXP_FIBERMAP (PR `#2100`_). +* Fix dtype of ``*_NSPEC`` columns to int16 (PR `#2103`_). +* desi_zcatalog --recoadd-fibermap option (PR `#2109`_, `#2116`_). +* fix HPXNEST header type (PR `#2110`_). +* desi_zcatalog --add-units option for DR1 patching (PR `#2111`_). +* Refactor desi_zcatalog options, add parallelism (PR `#2117`_). + +Algorithmic Updates + +* Fix incorrect weights in trace shift polyfit (PR `#2105`_). +* Improve trace shifts for bright/backup data (PR `#2106`_, `#2115`_). + +Miscellaneous + * Add read_spectra options to read subsets by targetids or rows (PR `#2052`_). +* Night QA flab calibs later than first science rather than absolute + timestamp (PR `#2089`_). +* Night QA orient images and display CCD amp names (PR `#2091`_). +* Add desi_compute_fiberflat_vs_humidity --first-night option (PR `#2101`_). +* Add desi_compute_gains exposure time flexibility (PR `#2107`_). +* Update readthedocs configuration (PR `#2112`_). .. _`#2052`: https://github.com/desihub/desispec/pull/2052 +.. _`#2089`: https://github.com/desihub/desispec/pull/2089 +.. _`#2090`: https://github.com/desihub/desispec/pull/2090 +.. _`#2091`: https://github.com/desihub/desispec/pull/2091 +.. _`#2093`: https://github.com/desihub/desispec/pull/2093 +.. _`#2094`: https://github.com/desihub/desispec/pull/2094 +.. _`#2100`: https://github.com/desihub/desispec/pull/2100 +.. _`#2101`: https://github.com/desihub/desispec/pull/2101 +.. _`#2103`: https://github.com/desihub/desispec/pull/2103 +.. _`#2105`: https://github.com/desihub/desispec/pull/2105 +.. _`#2106`: https://github.com/desihub/desispec/pull/2106 +.. _`#2107`: https://github.com/desihub/desispec/pull/2107 +.. _`#2109`: https://github.com/desihub/desispec/pull/2109 +.. _`#2110`: https://github.com/desihub/desispec/pull/2110 +.. _`#2111`: https://github.com/desihub/desispec/pull/2111 +.. _`#2112`: https://github.com/desihub/desispec/pull/2112 +.. _`#2114`: https://github.com/desihub/desispec/pull/2114 +.. _`#2115`: https://github.com/desihub/desispec/pull/2115 +.. _`#2116`: https://github.com/desihub/desispec/pull/2116 +.. _`#2117`: https://github.com/desihub/desispec/pull/2117 + 0.59.2 (2023-08-04) ------------------- From 91b9f3630ea5485676aa65e0e829063fd08cf557 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Thu, 14 Sep 2023 14:12:28 -0700 Subject: [PATCH 6/8] update release notes and version for desispec/0.60.0 --- py/desispec/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desispec/_version.py b/py/desispec/_version.py index ab048d1fe..575035ea1 100644 --- a/py/desispec/_version.py +++ b/py/desispec/_version.py @@ -1 +1 @@ -__version__ = '0.59.2.dev7913' +__version__ = '0.60.0' From 7a1792342861aa0ac43cd8d5836c8f2016cbbfc4 Mon Sep 17 00:00:00 2001 From: Stephen Bailey Date: Thu, 14 Sep 2023 14:12:46 -0700 Subject: [PATCH 7/8] update dev version after 0.60.0 tag --- py/desispec/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desispec/_version.py b/py/desispec/_version.py index 575035ea1..2912e374b 100644 --- a/py/desispec/_version.py +++ b/py/desispec/_version.py @@ -1 +1 @@ -__version__ = '0.60.0' +__version__ = '0.60.0.dev7988' From 56fd73f72b1d6a512e12b9afd5b254250fb98f7e Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Wed, 20 Sep 2023 10:34:10 -0700 Subject: [PATCH 8/8] get_surveys_night_expids(): protect against missing OBSTYPE keyword --- py/desispec/night_qa.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 9a16d77ac..0086c2674 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -105,6 +105,11 @@ def get_surveys_night_expids( expids, tileids, surveys = [], [], [] for i in range(len(fns)): hdr = fitsio.read_header(fns[i], "SPEC") + # AR protect against corrupted exposures... + # AR see https://github.com/desihub/desispec/issues/2119 + if "OBSTYPE" not in hdr: + log.warning("OBSTYPE keyword missing in {}; ignoring that exposure".format(fns[i])) + continue if hdr["OBSTYPE"] == "SCIENCE": survey = "unknown" # AR look for the fiberassign file