From 16abc8231d7824387ae4dd68dfafbe7206a2ea26 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 25 Aug 2017 10:40:32 -0700 Subject: [PATCH 01/10] clean up integration test code --- py/desispec/test/old_integration_test.py | 123 +++++++++++------------ 1 file changed, 60 insertions(+), 63 deletions(-) diff --git a/py/desispec/test/old_integration_test.py b/py/desispec/test/old_integration_test.py index aff366244..bee2c52f8 100644 --- a/py/desispec/test/old_integration_test.py +++ b/py/desispec/test/old_integration_test.py @@ -4,25 +4,25 @@ python -m desispec.test.integration_test """ from __future__ import absolute_import, print_function -import sys import os -import random import time import numpy as np from astropy.io import fits -from desispec.util import runcmd -from desispec import io -from desispec.qa import QA_Exposure +from ..util import runcmd +from .. import io +from ..qa import QA_Exposure from desiutil.log import get_logger #- prevent nose from trying to run this test since it takes too long __test__ = False def check_env(): - """ - Check required environment variables; raise RuntimeException if missing + """Check required environment variables. + + Raises: + RuntimeError if any script fails """ log = get_logger() #- template locations @@ -48,12 +48,8 @@ def check_env(): log.warning(" Raw data read from $DESI_SPECTRO_DATA/") log.warning(" Spectro pipeline output written to $DESI_SPECTRO_REDUX/$SPECPROD/") log.warning(" Templates are read from $DESI_BASIS_TEMPLATES") - - #- Wait until end to raise exception so that we report everything that - #- is missing before actually failing - if missing_env: log.critical("missing env vars; exiting without running pipeline") - sys.exit(1) + raise RuntimeError("missing env vars; exiting without running pipeline") #- Override $DESI_SPECTRO_DATA to match $DESI_SPECTRO_SIM/$PIXPROD os.environ['DESI_SPECTRO_DATA'] = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD')) @@ -61,7 +57,7 @@ def check_env(): #- TODO: fix usage of night to be something other than today def integration_test(night=None, nspec=5, clobber=False): - """Run an integration test from raw data simulations through redshifts + """Run an integration test from raw data simulations through redshifts. Args: night (str, optional): YEARMMDD, defaults to current night @@ -70,7 +66,6 @@ def integration_test(night=None, nspec=5, clobber=False): Raises: RuntimeError if any script fails - """ log = get_logger() #- YEARMMDD string, rolls over at noon not midnight @@ -85,8 +80,12 @@ def integration_test(night=None, nspec=5, clobber=False): #----- #- Input fibermaps, spectra, and pixel-level raw data - raw_dict = {0: 'flat', 1: 'arc', 2: 'dark'} - for expid, program in raw_dict.items(): + # raw_dict = {0: 'flat', 1: 'arc', 2: 'dark'} + programs = ('flat', 'arc', 'dark') + channels = ('b', 'r', 'z') + cameras = ('b0', 'r0', 'z0') + # for expid, program in raw_dict.items(): + for expid, program in enumerate(programs): cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, program=program, **params) @@ -101,7 +100,7 @@ def integration_test(night=None, nspec=5, clobber=False): inputs = [fibermap, simspec] outputs = list() outputs.append(fibermap.replace('fibermap-', 'simpix-')) - for camera in ['b0', 'r0', 'z0']: + for camera in cameras: pixfile = io.findfile('pix', night, expid, camera) outputs.append(pixfile) # outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-'))) @@ -111,18 +110,13 @@ def integration_test(night=None, nspec=5, clobber=False): #----- #- Extract - waverange = dict( - b = "3570,5940,1.0", - r = "5630,7740,1.0", - z = "7440,9830,1.0", - ) - for expid in [0,1,2]: - for channel in ['b', 'r', 'z']: - camera = channel+'0' - pixfile = io.findfile('pix', night, expid, camera) + waverange = dict(b="3570,5940,1.0", r="5630,7740,1.0", z="7440,9830,1.0") + for expid, program in enumerate(programs): + for ic, channel in enumerate(channels): + pixfile = io.findfile('pix', night, expid, cameras[ic]) fiberfile = io.findfile('fibermap', night, expid) psffile = '{}/data/specpsf/psf-{}.fits'.format(os.getenv('DESIMODEL'), channel) - framefile = io.findfile('frame', night, expid, camera) + framefile = io.findfile('frame', night, expid, cameras[ic]) # cmd = "exspec -i {pix} -p {psf} --specmin 0 --nspec {nspec} -w {wave} -o {frame}".format( # pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params) cmd = "desi_extract_spectra -i {pix} -p {psf} -f {fibermap} --specmin 0 --nspec {nspec} -o {frame}".format( @@ -131,43 +125,41 @@ def integration_test(night=None, nspec=5, clobber=False): inputs = [pixfile, psffile, fiberfile] outputs = [framefile,] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: - raise RuntimeError('extraction failed for {} expid {}'.format(camera, expid)) + raise RuntimeError('extraction failed for {} expid {}'.format(cameras[ic], expid)) #----- #- Fiber flat expid = 0 - for channel in ['b', 'r', 'z']: - camera = channel+"0" - framefile = io.findfile('frame', night, expid, camera) - fiberflat = io.findfile('fiberflat', night, expid, camera) + for ic, channel in enumerate(channels): + framefile = io.findfile('frame', night, expid, cameras[ic]) + fiberflat = io.findfile('fiberflat', night, expid, cameras[ic]) fibermap = io.findfile('fibermap', night, expid) # for QA - qafile = io.findfile('qa_calib', night, expid, camera) - qafig = io.findfile('qa_flat_fig', night, expid, camera) + qafile = io.findfile('qa_calib', night, expid, cameras[ic]) + qafig = io.findfile('qa_flat_fig', night, expid, cameras[ic]) cmd = "desi_compute_fiberflat --infile {frame} --outfile {fiberflat} --qafile {qafile} --qafig {qafig}".format( frame=framefile, fiberflat=fiberflat, qafile=qafile, qafig=qafig, **params) inputs = [framefile,fibermap,] outputs = [fiberflat,qafile,qafig,] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: - raise RuntimeError('fiberflat failed for '+camera) + raise RuntimeError('fiberflat failed for '+cameras[ic]) #----- #- Sky model flat_expid = 0 expid = 2 - for channel in ['b', 'r', 'z']: - camera = channel+"0" - framefile = io.findfile('frame', night, expid, camera) + for ic, channel in enumerate(channels): + framefile = io.findfile('frame', night, expid, cameras[ic]) fibermap = io.findfile('fibermap', night, expid) - fiberflat = io.findfile('fiberflat', night, flat_expid, camera) - skyfile = io.findfile('sky', night, expid, camera) - qafile = io.findfile('qa_data', night, expid, camera) - qafig = io.findfile('qa_sky_fig', night, expid, camera) + fiberflat = io.findfile('fiberflat', night, flat_expid, cameras[ic]) + skyfile = io.findfile('sky', night, expid, cameras[ic]) + qafile = io.findfile('qa_data', night, expid, cameras[ic]) + qafig = io.findfile('qa_sky_fig', night, expid, cameras[ic]) cmd="desi_compute_sky --infile {frame} --fiberflat {fiberflat} --outfile {sky} --qafile {qafile} --qafig {qafig}".format( frame=framefile, fiberflat=fiberflat, sky=skyfile, qafile=qafile, qafig=qafig, **params) inputs = [framefile, fibermap, fiberflat] outputs = [skyfile, qafile, qafig,] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: - raise RuntimeError('sky model failed for '+camera) + raise RuntimeError('sky model failed for '+cameras[ic]) #----- @@ -181,11 +173,10 @@ def integration_test(night=None, nspec=5, clobber=False): flats = list() frames = list() skymodels = list() - for channel in ['b', 'r', 'z']: - camera = channel+'0' - frames.append( io.findfile('frame', night, expid, camera) ) - flats.append( io.findfile('fiberflat', night, flat_expid, camera) ) - skymodels.append( io.findfile('sky', night, expid, camera) ) + for ic, channel in enumerate(channels): + frames.append( io.findfile('frame', night, expid, cameras[ic]) ) + flats.append( io.findfile('fiberflat', night, flat_expid, cameras[ic]) ) + skymodels.append( io.findfile('sky', night, expid, cameras[ic]) ) frames = ' '.join(frames) flats = ' '.join(flats) @@ -208,15 +199,14 @@ def integration_test(night=None, nspec=5, clobber=False): #----- #- Flux calibration - for channel in ['b', 'r', 'z']: - camera = channel+"0" - framefile = io.findfile('frame', night, expid, camera) + for ic, channel in enumerate(channels): + framefile = io.findfile('frame', night, expid, cameras[ic]) fibermap = io.findfile('fibermap', night, expid) - fiberflat = io.findfile('fiberflat', night, flat_expid, camera) - skyfile = io.findfile('sky', night, expid, camera) - calibfile = io.findfile('calib', night, expid, camera) - qafile = io.findfile('qa_data', night, expid, camera) - qafig = io.findfile('qa_flux_fig', night, expid, camera) + fiberflat = io.findfile('fiberflat', night, flat_expid, cameras[ic]) + skyfile = io.findfile('sky', night, expid, cameras[ic]) + calibfile = io.findfile('calib', night, expid, cameras[ic]) + qafile = io.findfile('qa_data', night, expid, cameras[ic]) + qafig = io.findfile('qa_flux_fig', night, expid, cameras[ic]) #- Compute flux calibration vector cmd = """desi_compute_fluxcalibration \ @@ -228,10 +218,10 @@ def integration_test(night=None, nspec=5, clobber=False): inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile] outputs = [calibfile, qafile, qafig] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: - raise RuntimeError('flux calibration failed for '+camera) + raise RuntimeError('flux calibration failed for '+cameras[ic]) #- Apply the flux calibration to write a cframe file - cframefile = io.findfile('cframe', night, expid, camera) + cframefile = io.findfile('cframe', night, expid, cameras[ic]) cmd = """desi_process_exposure \ --infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \ --outfile {cframe}""".format(frame=framefile, fibermap=fibermap, @@ -239,7 +229,7 @@ def integration_test(night=None, nspec=5, clobber=False): inputs = [framefile, fiberflat, skyfile, calibfile] outputs = [cframefile, ] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: - raise RuntimeError('combining calibration steps failed for '+camera) + raise RuntimeError('combining calibration steps failed for '+cameras[ic]) #----- #- Collate QA @@ -251,7 +241,7 @@ def integration_test(night=None, nspec=5, clobber=False): expid = 2 qafile = io.findfile('qa_data_exp', night, expid) if clobber or not os.path.exists(qafile): - flavor = program2flavor[raw_dict[expid]] + flavor = program2flavor[programs[expid]] qaexp_data = QA_Exposure(expid, night, flavor) # Removes camera files io.write_qa_exposure(os.path.splitext(qafile)[0], qaexp_data) if not os.path.exists(qafile): @@ -261,7 +251,7 @@ def integration_test(night=None, nspec=5, clobber=False): for expid in calib_expid: qafile = io.findfile('qa_calib_exp', night, expid) if clobber or not os.path.exists(qafile): - qaexp_calib = QA_Exposure(expid, night, raw_dict[expid]) + qaexp_calib = QA_Exposure(expid, night, programs[expid]) io.write_qa_exposure(os.path.splitext(qafile)[0], qaexp_calib) if not os.path.exists(qafile): raise RuntimeError('FAILED calib QA_Exposure({},{}, ...) -> {}'.format(expid, night, qafile)) @@ -270,7 +260,7 @@ def integration_test(night=None, nspec=5, clobber=False): #- Regroup cframe -> spectra expid = 2 inputs = list() - for camera in ['b0', 'r0', 'z0']: + for camera in cameras: inputs.append( io.findfile('cframe', night, expid, camera) ) outputs = list() @@ -358,5 +348,12 @@ def integration_test(night=None, nspec=5, clobber=False): print("--------------------------------------------------") + if __name__ == '__main__': - integration_test() + from sys import exit + status = 0 + try: + integration_test() + except RuntimeError: + status = 1 + exit(status) From 96a95af1ffe3f1b2395616b4ec22eb8fcd243fdf Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 25 Aug 2017 11:48:41 -0700 Subject: [PATCH 02/10] load zbest files --- py/desispec/database/datachallenge.py | 140 +++++++++++++++-------- py/desispec/test/old_integration_test.py | 14 ++- 2 files changed, 104 insertions(+), 50 deletions(-) diff --git a/py/desispec/database/datachallenge.py b/py/desispec/database/datachallenge.py index 7b3fa3100..c4cf7f3bb 100644 --- a/py/desispec/database/datachallenge.py +++ b/py/desispec/database/datachallenge.py @@ -540,61 +540,25 @@ def q3c_index(table): return -def main(): - """Entry point for command-line script. +def setup_db(options): + """Initialize the database connection. + + Parameters + ---------- + options : :class:`argpare.Namespace` + Parsed command-line options. Returns ------- - :class:`int` - An integer suitable for passing to :func:`sys.exit`. + :class:`bool` + ``True`` if the configured database is a PostgreSQL database. """ global engine, schemaname - from os import remove from os.path import basename, exists, join - from sys import argv - from argparse import ArgumentParser - from pkg_resources import resource_filename - from pytz import utc - from desiutil.log import get_logger, DEBUG, INFO - # - # command-line arguments - # - prsr = ArgumentParser(description=("Load a data challenge simulation into a " + - "database."), - prog=basename(argv[0])) - prsr.add_argument('-c', '--clobber', action='store_true', dest='clobber', - help='Delete any existing file(s) before loading.') - prsr.add_argument('-f', '--filename', action='store', dest='dbfile', - default='quicksurvey.db', metavar='FILE', - help="Store data in FILE.") - prsr.add_argument('-H', '--hostname', action='store', dest='hostname', - metavar='HOSTNAME', - help='If specified, connect to a PostgreSQL database on HOSTNAME.') - prsr.add_argument('-m', '--max-rows', action='store', dest='maxrows', - type=int, default=0, metavar='M', - help="Load up to M rows in the tables (default is all rows).") - prsr.add_argument('-r', '--rows', action='store', dest='chunksize', - type=int, default=50000, metavar='N', - help="Load N rows at a time (default %(default)s).") - prsr.add_argument('-s', '--schema', action='store', dest='schema', - metavar='SCHEMA', - help='Set the schema name in the PostgreSQL database.') - prsr.add_argument('-U', '--username', action='store', dest='username', - metavar='USERNAME', default='desidev_admin', - help="If specified, connect to a PostgreSQL database with USERNAME.") - prsr.add_argument('-v', '--verbose', action='store_true', dest='verbose', - help='Print extra information.') - prsr.add_argument('datapath', metavar='DIR', help='Load the data in DIR.') - options = prsr.parse_args() - # - # Logging - # - if options.verbose: - log = get_logger(DEBUG, timestamp=True) - else: - log = get_logger(INFO, timestamp=True) + from desiutil.log import get_logger + log = get_logger() # - # Schema. + # Schema creation # if options.schema: schemaname = options.schema @@ -635,6 +599,86 @@ def main(): tab.schema = schemaname Base.metadata.create_all(engine) log.info("Finished creating tables.") + return postgresql + + +def get_options(*args): + """Parse command-line options. + + Parameters + ---------- + args : iterable + If arguments are passed, use them instead of ``sys.argv``. + + Returns + ------- + :class:`argparse.Namespace` + The parsed options. + """ + from sys import argv + from os.path import basename + from argparse import ArgumentParser + prsr = ArgumentParser(description=("Load a data challenge simulation into a " + + "database."), + prog=basename(argv[0])) + prsr.add_argument('-c', '--clobber', action='store_true', dest='clobber', + help='Delete any existing file(s) before loading.') + prsr.add_argument('-f', '--filename', action='store', dest='dbfile', + default='quicksurvey.db', metavar='FILE', + help="Store data in FILE.") + prsr.add_argument('-H', '--hostname', action='store', dest='hostname', + metavar='HOSTNAME', + help='If specified, connect to a PostgreSQL database on HOSTNAME.') + prsr.add_argument('-m', '--max-rows', action='store', dest='maxrows', + type=int, default=0, metavar='M', + help="Load up to M rows in the tables (default is all rows).") + prsr.add_argument('-r', '--rows', action='store', dest='chunksize', + type=int, default=50000, metavar='N', + help="Load N rows at a time (default %(default)s).") + prsr.add_argument('-s', '--schema', action='store', dest='schema', + metavar='SCHEMA', + help='Set the schema name in the PostgreSQL database.') + prsr.add_argument('-U', '--username', action='store', dest='username', + metavar='USERNAME', default='desidev_admin', + help="If specified, connect to a PostgreSQL database with USERNAME.") + prsr.add_argument('-v', '--verbose', action='store_true', dest='verbose', + help='Print extra information.') + prsr.add_argument('datapath', metavar='DIR', help='Load the data in DIR.') + if len(args) > 0: + options = prsr.parse_args(args) + else: + options = prsr.parse_args() + return options + + +def main(): + """Entry point for command-line script. + + Returns + ------- + :class:`int` + An integer suitable for passing to :func:`sys.exit`. + """ + from os import remove + from os.path import join + # from pkg_resources import resource_filename + from pytz import utc + from desiutil.log import get_logger, DEBUG, INFO + # + # command-line arguments + # + options = get_options() + # + # Logging + # + if options.verbose: + log = get_logger(DEBUG, timestamp=True) + else: + log = get_logger(INFO, timestamp=True) + # + # Initialize DB + # + postgresql = setup_db(options) # # Load configuration # diff --git a/py/desispec/test/old_integration_test.py b/py/desispec/test/old_integration_test.py index bee2c52f8..8bde69ef3 100644 --- a/py/desispec/test/old_integration_test.py +++ b/py/desispec/test/old_integration_test.py @@ -13,6 +13,8 @@ from ..util import runcmd from .. import io from ..qa import QA_Exposure +from ..database.datachallenge import setup_db, load_zcat + from desiutil.log import get_logger #- prevent nose from trying to run this test since it takes too long @@ -201,7 +203,7 @@ def integration_test(night=None, nspec=5, clobber=False): #- Flux calibration for ic, channel in enumerate(channels): framefile = io.findfile('frame', night, expid, cameras[ic]) - fibermap = io.findfile('fibermap', night, expid) + fibermap = io.findfile('fibermap', night, expid)join fiberflat = io.findfile('fiberflat', night, flat_expid, cameras[ic]) skyfile = io.findfile('sky', night, expid, cameras[ic]) calibfile = io.findfile('calib', night, expid, cameras[ic]) @@ -229,7 +231,7 @@ def integration_test(night=None, nspec=5, clobber=False): inputs = [framefile, fiberflat, skyfile, calibfile] outputs = [cframefile, ] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: - raise RuntimeError('combining calibration steps failed for '+cameras[ic]) + raise RuntimeError('combining calibration steps failed for 'join+cameras[ic]) #----- #- Collate QA @@ -286,6 +288,14 @@ def integration_test(night=None, nspec=5, clobber=False): if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('rrdesi failed for healpixel {}'.format(pix)) + # + # Load redshifts into database + # + options = get_options('--clobber', '--filename', 'dailytest.db', + os.path.join(os.environ['DESI_SPECTRO_REDUX'], + os.environ['SPECPROD'])) + postgresql = setup_db(options) + load_zcat(options.datapath) # ztruth QA # qafile = io.findfile('qa_ztruth', night) # qafig = io.findfile('qa_ztruth_fig', night) From 5dbae3005b6d633db780fa285c79386dc24ec616 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 25 Aug 2017 11:50:10 -0700 Subject: [PATCH 03/10] fix syntax [skip ci] --- py/desispec/test/old_integration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desispec/test/old_integration_test.py b/py/desispec/test/old_integration_test.py index 8bde69ef3..48c8f31ee 100644 --- a/py/desispec/test/old_integration_test.py +++ b/py/desispec/test/old_integration_test.py @@ -203,7 +203,7 @@ def integration_test(night=None, nspec=5, clobber=False): #- Flux calibration for ic, channel in enumerate(channels): framefile = io.findfile('frame', night, expid, cameras[ic]) - fibermap = io.findfile('fibermap', night, expid)join + fibermap = io.findfile('fibermap', night, expid) fiberflat = io.findfile('fiberflat', night, flat_expid, cameras[ic]) skyfile = io.findfile('sky', night, expid, cameras[ic]) calibfile = io.findfile('calib', night, expid, cameras[ic]) From 555839deb4c425d01dd5c0df9c4233b9d4188448 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 25 Aug 2017 11:50:58 -0700 Subject: [PATCH 04/10] fix syntax [skip ci] --- py/desispec/test/old_integration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desispec/test/old_integration_test.py b/py/desispec/test/old_integration_test.py index 48c8f31ee..957a6b258 100644 --- a/py/desispec/test/old_integration_test.py +++ b/py/desispec/test/old_integration_test.py @@ -231,7 +231,7 @@ def integration_test(night=None, nspec=5, clobber=False): inputs = [framefile, fiberflat, skyfile, calibfile] outputs = [cframefile, ] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: - raise RuntimeError('combining calibration steps failed for 'join+cameras[ic]) + raise RuntimeError('combining calibration steps failed for '+cameras[ic]) #----- #- Collate QA From c86aa6588fca04c4e2560b1c1b5b354c98449106 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 25 Aug 2017 12:05:58 -0700 Subject: [PATCH 05/10] fix import [skip ci] --- py/desispec/test/old_integration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desispec/test/old_integration_test.py b/py/desispec/test/old_integration_test.py index 957a6b258..ed060aec9 100644 --- a/py/desispec/test/old_integration_test.py +++ b/py/desispec/test/old_integration_test.py @@ -13,7 +13,7 @@ from ..util import runcmd from .. import io from ..qa import QA_Exposure -from ..database.datachallenge import setup_db, load_zcat +from ..database.datachallenge import get_options, setup_db, load_zcat from desiutil.log import get_logger From 3ac0eb1cfa0fb895b40e4c354fed00116886fc2c Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 25 Aug 2017 12:12:45 -0700 Subject: [PATCH 06/10] change data path [skip ci] --- py/desispec/database/datachallenge.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/py/desispec/database/datachallenge.py b/py/desispec/database/datachallenge.py index c4cf7f3bb..f74b84da2 100644 --- a/py/desispec/database/datachallenge.py +++ b/py/desispec/database/datachallenge.py @@ -389,7 +389,9 @@ def load_zcat(datapath, run1d='dc17a2', q3c=False): from astropy.io import fits from desiutil.log import get_logger log = get_logger() - zbestpath = join(datapath, 'spectro', 'redux', run1d, 'spectra-64', + # zbestpath = join(datapath, 'spectro', 'redux', run1d, 'spectra-64', + # '*', '*', 'zbest-64-*.fits') + zbestpath = join(datapath, 'spectra-64', '*', '*', 'zbest-64-*.fits') log.info("Using zbest file search path: %s.", zbestpath) zbest_files = glob(zbestpath) From 7623fc41e6d08b90011b50e899e2f6c3bc8f85e2 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 25 Aug 2017 12:17:08 -0700 Subject: [PATCH 07/10] fix import [skip ci] --- py/desispec/database/datachallenge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desispec/database/datachallenge.py b/py/desispec/database/datachallenge.py index f74b84da2..45cd1a01e 100644 --- a/py/desispec/database/datachallenge.py +++ b/py/desispec/database/datachallenge.py @@ -556,6 +556,7 @@ def setup_db(options): ``True`` if the configured database is a PostgreSQL database. """ global engine, schemaname + from os import remove from os.path import basename, exists, join from desiutil.log import get_logger log = get_logger() @@ -661,7 +662,6 @@ def main(): :class:`int` An integer suitable for passing to :func:`sys.exit`. """ - from os import remove from os.path import join # from pkg_resources import resource_filename from pytz import utc From 22d733fca9521538e80514f848758f2cfce6c4c9 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 30 Aug 2017 14:27:41 -0700 Subject: [PATCH 08/10] prepare to load end2end test data --- py/desispec/database/datachallenge.py | 44 ++++++++++++++------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/py/desispec/database/datachallenge.py b/py/desispec/database/datachallenge.py index 45cd1a01e..c0879f62e 100644 --- a/py/desispec/database/datachallenge.py +++ b/py/desispec/database/datachallenge.py @@ -173,33 +173,35 @@ class ObsList(SchemaMixin, Base): """Representation of the obslist table. """ - mjd = Column(Float, nullable=False) - exptime = Column(Float, nullable=False) - program = Column(String, nullable=False) - passnum = Column(Integer, nullable=False) tileid = Column(Integer, primary_key=True, autoincrement=False) + passnum = Column(Integer, nullable=False) ra = Column(Float, nullable=False) dec = Column(Float, nullable=False) - moonfrac = Column(Float, nullable=False) - moondist = Column(Float, nullable=False) - moonalt = Column(Float, nullable=False) + night = Column(String, nullable=False) + mjd = Column(Float, nullable=False) + exptime = Column(Float, nullable=False) seeing = Column(Float, nullable=False) airmass = Column(Float, nullable=False) + # program = Column(String, nullable=False) + moonfrac = Column(Float, nullable=False) + moonalt = Column(Float, nullable=False) + moonsep = Column(Float, nullable=False) # dateobs = Column(DateTime(timezone=True), nullable=False) def __repr__(self): - return ("").format(self) + "moonsep={0.moonsep:f}" + + ")>").format(self) class ZCat(SchemaMixin, Base): @@ -389,10 +391,10 @@ def load_zcat(datapath, run1d='dc17a2', q3c=False): from astropy.io import fits from desiutil.log import get_logger log = get_logger() - # zbestpath = join(datapath, 'spectro', 'redux', run1d, 'spectra-64', - # '*', '*', 'zbest-64-*.fits') - zbestpath = join(datapath, 'spectra-64', + zbestpath = join(datapath, 'spectro', 'redux', run1d, 'spectra-64', '*', '*', 'zbest-64-*.fits') + # zbestpath = join(datapath, 'spectra-64', + # '*', '*', 'zbest-64-*.fits') log.info("Using zbest file search path: %s.", zbestpath) zbest_files = glob(zbestpath) if len(zbest_files) == 0: @@ -462,7 +464,7 @@ def load_fiberassign(datapath, maxpass=4, q3c=False, latest_epoch=False): from astropy.io import fits from desiutil.log import get_logger log = get_logger() - fiberpath = join(datapath, 'fiberassign', 'output', + fiberpath = join(datapath, 'fiberassign', 'tile_*.fits') log.info("Using tile file search path: %s.", fiberpath) tile_files = glob(fiberpath) @@ -712,7 +714,7 @@ def main(): 'q3c': postgresql, 'chunksize': options.chunksize, 'maxrows': options.maxrows}, - {'filepath': join(options.datapath, 'twopct.ecsv'), + {'filepath': join(options.datapath, 'survey', 'exposures.fits'), 'tcls': ObsList, 'hdu': 1, 'expand': {'PASS': 'passnum'}, @@ -750,7 +752,7 @@ def main(): q = dbSession.query(ZCat).first() if q is None: log.info("Loading ZCat from %s.", options.datapath) - load_zcat(options.datapath) + load_zcat(options.datapath, run1d='mini') log.info("Finished loading ZCat.") else: log.info("ZCat table already loaded.") From f014f94591514461a104d4679d38ff49b22fad37 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 30 Aug 2017 14:40:55 -0700 Subject: [PATCH 09/10] skip loading obslist table --- py/desispec/database/datachallenge.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/py/desispec/database/datachallenge.py b/py/desispec/database/datachallenge.py index c0879f62e..3ea4e006b 100644 --- a/py/desispec/database/datachallenge.py +++ b/py/desispec/database/datachallenge.py @@ -713,16 +713,16 @@ def main(): 'convert': None, 'q3c': postgresql, 'chunksize': options.chunksize, - 'maxrows': options.maxrows}, - {'filepath': join(options.datapath, 'survey', 'exposures.fits'), - 'tcls': ObsList, - 'hdu': 1, - 'expand': {'PASS': 'passnum'}, - # 'convert': {'dateobs': lambda x: convert_dateobs(x, tzinfo=utc)}, - 'convert': None, - 'q3c': postgresql, - 'chunksize': options.chunksize, 'maxrows': options.maxrows},] + # {'filepath': join(options.datapath, 'survey', 'exposures.fits'), + # 'tcls': ObsList, + # 'hdu': 1, + # 'expand': {'PASS': 'passnum'}, + # # 'convert': {'dateobs': lambda x: convert_dateobs(x, tzinfo=utc)}, + # 'convert': None, + # 'q3c': postgresql, + # 'chunksize': options.chunksize, + # 'maxrows': options.maxrows},] # {'filepath': join(options.datapath, 'output', 'dark', '4', 'zcat.fits'), # 'tcls': ZCat, # 'hdu': 1, From 1e3a1ecd05ecae186e6063381edd615a32dbc707 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Wed, 30 Aug 2017 14:49:41 -0700 Subject: [PATCH 10/10] disable targetid integrity check (and pray) --- py/desispec/database/datachallenge.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/py/desispec/database/datachallenge.py b/py/desispec/database/datachallenge.py index 3ea4e006b..6ed4a2e6e 100644 --- a/py/desispec/database/datachallenge.py +++ b/py/desispec/database/datachallenge.py @@ -410,12 +410,16 @@ def load_zcat(datapath, run1d='dc17a2', q3c=False): data = hdulist[1].data log.info("Read data from %s.", f) good_targetids = data['TARGETID'] != 0 - q = dbSession.query(ZCat).filter(ZCat.targetid.in_(data['TARGETID'].tolist())).all() - if len(q) != 0: - log.warning("Duplicate TARGETID found in %s.", f) - for z in q: - log.warning("Duplicate TARGETID = %d.", z.targetid) - good_targetids = good_targetids & (data['TARGETID'] != z.targetid) + # + # If there are too many targetids, the in_ clause will blow up. + # Disabling this test, and crossing fingers. + # + # q = dbSession.query(ZCat).filter(ZCat.targetid.in_(data['TARGETID'].tolist())).all() + # if len(q) != 0: + # log.warning("Duplicate TARGETID found in %s.", f) + # for z in q: + # log.warning("Duplicate TARGETID = %d.", z.targetid) + # good_targetids = good_targetids & (data['TARGETID'] != z.targetid) data_list = [data[col][good_targetids].tolist() for col in data.names if col != 'COEFF'] data_names = [col.lower() for col in data.names if col != 'COEFF']