Merge pull request #113 from SgmAstro/survey_nbs

Survey nbs

docs/nb/sandbox/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/nb/sandbox/.ipynb_checkpoints/Untitled1-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

findfile.py

@@ -14,6 +14,7 @@
              'zmax',\
              'vmax',\
              'lumfn',\
+             'lumfn_step',\
              'ddp',\
              'ddp_n8']
 
@@ -43,15 +44,18 @@ def fetch_fields(survey):
 
     return fields
 
-def release_dir(user=os.environ['USER'], versioned=True, survey='gama'):
-    assert survey == 'gama', 'TODO: Support DESI.'
+def release_dir(user=os.environ['USER'], survey='gama', version=None):
+    #assert survey == 'gama', 'TODO: Support DESI.'
 
     # E.g.  /cosma/home/durham/dc-wils7/data/GAMA4/                                                                                                                                                
-    if versioned:
+    if version == 'latest':
         ff = glob.glob('/cosma/home/durham/{}/data/v*'.format(user))
         ff.sort(key=os.path.getmtime)
 
         return  ff[-1]
+
+    elif version != None:
+        return '/cosma/home/durham/{}/data/{}/'.format(user, version)
 
     else:
         return '/cosma/home/durham/{}/data/GAMA4/'.format(user)
@@ -61,9 +65,9 @@ def overwrite_check(opath):
         print('{} found on disk and overwrite forbidden (--nooverwrite).'.format(opath))
         exit(0)
 
-def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', survey='gama', realz=0, debug=False):    
+def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', survey='gama', realz=0, debug=False, version=None):    
     survey = survey.lower()
-
+    
     # Special case:                                                                                                                                                                                 
     if (ftype == 'gold') & dryrun & (survey == 'gama'):
         return  os.environ['CODE_ROOT'] + '/data/gama_gold_dryrun.fits'
@@ -75,21 +79,47 @@ def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', surv
         fields = desi_fields
 
     else:
-        raise NotImplementedError
+        raise NotImplementedError()
+
+    # TODO: Re-add assert and test
+    #if field != None:
+    #    assert field in fields
 
     if dryrun:
         dryrun = '_dryrun'
         debug  = True
+
     else:
         dryrun = ''
 
     realz      = str(realz)
 
-    gold_dir   = os.environ['GOLD_DIR']
-    rand_dir   = os.environ['RANDOMS_DIR']
+    if version == None:        
+        if 'GOLD_DIR' in os.environ:
+            gold_dir = os.environ['GOLD_DIR']
+
+        else:
+            gold_dir = os.environ['HOME'] + '/data/GAMA4/'
 
+            print('Warning:  GOLD_DIR not defined in environment; assuming {gold_dir}')
+
+        if 'RANDOMS_DIR' in os.environ:
+            rand_dir = os.environ['RANDOMS_DIR']
+
+        else:
+            rand_dir = os.environ['HOME'] + '/data/GAMA4/randoms/'
+
+            print('Warning:  RANDOMS_DIR not defined in environment; assuming {randoms_dir}')
+
+    else:
+        gold_dir = release_dir(version=version)
+        rand_dir = release_dir(version=version) + '/randoms/'
+
     if isinstance(field, list):
         return  [findfile(ftype, dryrun=dryrun, prefix=prefix, field=ff, utier=utier) for ff in field]
+
+    if ftype == 'summary_log':
+        return gold_dir + 'summary.log'
 
     if field == None:
         file_types = {'gold':       {'dir': gold_dir, 'id': f'{survey}',      'ftype': 'gold'},\
@@ -105,34 +135,36 @@ def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', surv
         fpath      = parts['dir'] + '/{}_{}{}.fits'.format(parts['id'], parts['ftype'], dryrun)
 
     else: 
-        file_types = {'ddp_n8_d0':           {'dir': gold_dir, 'id': f'{survey}_gold',         'ftype': 'ddp_n8_d0_{}'.format(utier)},\
-                      'ddp_n8_d0_vmax':      {'dir': gold_dir, 'id': f'{survey}_gold',         'ftype': 'ddp_n8_d0_{}_vmax'.format(utier)},\
-                      'ddp_n8_d0_lumfn':     {'dir': gold_dir, 'id': f'{survey}_gold',         'ftype': 'ddp_n8_d0_{}_lumfn'.format(utier)},\
-                      'randoms':             {'dir': rand_dir, 'id': 'randoms',                'ftype': realz},\
-                      'randoms_n8':          {'dir': rand_dir, 'id': 'randoms_N8',             'ftype': realz},\
-                      'randoms_bd':          {'dir': rand_dir, 'id': 'randoms_bd',             'ftype': realz},\
-                      'randoms_bd_ddp_n8':   {'dir': rand_dir, 'id': 'randoms_bd_ddp_n8',      'ftype': realz}
+        file_types = {'ddp_n8_d0':          {'dir': gold_dir, 'id': f'{survey}_gold',         'ftype': 'ddp_n8_d0_{}'.format(utier)},\
+                      'ddp_n8_d0_vmax':     {'dir': gold_dir, 'id': f'{survey}_gold',         'ftype': 'ddp_n8_d0_{}_vmax'.format(utier)},\
+                      'ddp_n8_d0_lumfn':    {'dir': gold_dir, 'id': f'{survey}_gold',         'ftype': 'ddp_n8_d0_{}_lumfn'.format(utier)},\
+                      'randoms':            {'dir': rand_dir, 'id': 'randoms',                'ftype': realz},\
+                      'randoms_n8':         {'dir': rand_dir, 'id': 'randoms_N8',             'ftype': realz},\
+                      'randoms_bd':         {'dir': rand_dir, 'id': 'randoms_bd',             'ftype': realz},\
+                      'randoms_bd_ddp_n8':  {'dir': rand_dir, 'id': 'randoms_bd_ddp_n8',      'ftype': realz}
                      }
 
         parts      = file_types[ftype]
         fpath      = f'' + parts['dir'] + '/{}_{}_{}{}.fits'.format(parts['id'], field, parts['ftype'], dryrun)
 
-        if prefix != None:
-            assert 'randoms' in prefix;
-
-            dirname = os.path.dirname(fpath)
-            fpath = os.path.basename(fpath)
+    if prefix != None:
+        assert 'randoms' in prefix;
+        assert 'randoms' in fpath
+
+        dirname = os.path.dirname(fpath)
+        fpath   = os.path.basename(fpath)
 
-            fpath = fpath.replace('randoms', prefix)
-            fpath = dirname + '/' + fpath
-
+        fpath   = fpath.replace('randoms', prefix)
+        fpath   = dirname + '/' + fpath
 
     if debug:
         print(f'DEBUG: findfile returns {fpath}')
+
+    fpath = fpath.replace('//', '/')
 
     return  fpath
 
-def file_check(dryrun=None):
+def file_check(dryrun=None):        
     try:
         dryrun = os.environ['DRYRUN']
 
@@ -141,24 +173,26 @@ def file_check(dryrun=None):
 
         dryrun = ''
 
-    fpaths = [findfile(xx, dryrun=False, prefix='', field=None) for xx in supported]
-
-    for field in fields:
-        fpaths.append(findfile('randoms',            dryrun=False, prefix='', field=field))
-        fpaths.append(findfile('randoms_n8',         dryrun=False, prefix='', field=field))
-        fpaths.append(findfile('randoms_bd',         dryrun=False, prefix='', field=field))
-        fpaths.append(findfile('randoms_ddp1',       dryrun=False, prefix='', field=field))
-        fpaths.append(findfile('randoms_ddp1_n8',    dryrun=False, prefix='', field=field))
-        fpaths.append(findfile('randoms_ddp1_bd',    dryrun=False, prefix='', field=field))
-        fpaths.append(findfile('randoms_ddp1_bd_n8', dryrun=False, prefix='', field=field))
-        fpaths.append(findfile('randoms_bd_ddp_n8',  dryrun=False, prefix='', field=field))
-
-        for ii, _ in enumerate(d8_limits):
-            fpaths.append(findfile('ddp_n8_d0',       dryrun=False, prefix='', field=field, utier=ii))
-            fpaths.append(findfile('ddp_n8_d0_vmax',  dryrun=False, prefix='', field=field, utier=ii))
-            fpaths.append(findfile('ddp_n8_d0_lumfn', dryrun=False, prefix='', field=field, utier=ii))
-
-
+    fpaths = []
+
+    for survey in ['desi', 'gama']:
+        for xx in supported:
+            fpaths.append(findfile(xx, dryrun=False, survey=survey))
+
+        fields  = fetch_fields(survey)
+
+        for field in fields:
+            for prefix in [None, 'randoms_ddp1']:
+                fpaths.append(findfile('randoms',            dryrun=False, field=field, prefix=prefix))
+                fpaths.append(findfile('randoms_n8',         dryrun=False, field=field, prefix=prefix))
+                fpaths.append(findfile('randoms_bd',         dryrun=False, field=field, prefix=prefix))
+                fpaths.append(findfile('randoms_bd_ddp_n8',  dryrun=False, field=field, prefix=prefix))
+
+            for ii, _ in enumerate(d8_limits):
+                fpaths.append(findfile('ddp_n8_d0',       dryrun=False, field=field, utier=ii, survey=survey))
+                fpaths.append(findfile('ddp_n8_d0_vmax',  dryrun=False, field=field, utier=ii, survey=survey))
+                fpaths.append(findfile('ddp_n8_d0_lumfn', dryrun=False, field=field, utier=ii, survey=survey))
+
     print('\n\n----  SUPPORTED FPATHS    ----\n')
 
     for fp in fpaths:
@@ -175,7 +209,8 @@ def file_check(dryrun=None):
     rand_paths  = sorted(glob.glob(os.environ['RANDOMS_DIR'] + '/*.fits'))
     all_paths   = gold_paths + rand_paths
 
-    unsupported = [x for x in all_paths if (x not in fpaths and 'dryrun' not in x)]
+    unsupported = [x for x in all_paths if x not in fpaths]
+    unsupported = [x for x in unsupported if 'dryrun' not in x]
 
     print('\n\n----  UNSUPPORTED FPATHS    ----\n')
 
@@ -195,5 +230,5 @@ def file_check(dryrun=None):
 
 if __name__ == '__main__':
     failure = file_check()
-
+    
     print('\n\nSuccess: {}\n\n'.format(~failure))

gama_gold.py

@@ -98,6 +98,9 @@ def gama_gold(args):
     idx   = np.random.choice(np.arange(len(dat)), 5000, replace=False)
     dat   = dat[idx]
 
+    dat.meta = dat.meta = {'AREA': dat.meta['AREA'],\
+                           'GOLD_NGAL': dat.meta['GOLD_NGAL']}
+
     dat.write(os.environ['CODE_ROOT'] + '/data/gama_gold_dryrun.fits', format='fits', overwrite=True)
 
 

test_allnb.py

@@ -1,41 +1,41 @@
 import argparse
 import papermill as pm
 
-from   tidyup   import   tidyup
+from   tidyup   import tidyup
 from   findfile import fetch_fields
 
-parser  = argparse.ArgumentParser(description='Select field.')
-parser.add_argument('-s', '--survey', help='Survey, e.g. GAMA, DESI, etc.', type=str, default='gama')
-
-survey  = args.survey.lower()
-
-fields = fetchfields(survey)
-
 # https://docs.pytest.org/en/6.2.x/
-def test_allnbs():
+def test_allnbs(survey='gama'):
     print('Running all tests.')
-
+
+    if (survey != 'gama') and (survey != 'desi'):
+        raise  NotImplementedError(f'No implementation for survey: {survey}')
+
     tidyup()
 
-    run_randomqa()
-
-    run_goldqa()
+    run_randomqa(survey)
 
-    run_delta8qa()
+    run_goldqa(survey)
+
+    run_delta8qa(survey)
 
     print('Done.')
 
-def run_randomqa():
+def run_randomqa(survey):
+    fields = fetch_fields(survey)
+
+    print(fields)
+
     for field in fields:
         print('Running random QA for field {}'.format(field))
-        
+
         pm.execute_notebook('docs/nb/randoms_n8_qa.ipynb',\
                             'test/pm_randoms_n8_{}_qa.ipynb'.format(field),\
-                            parameters=dict(field=field),\
+                            parameters=dict(field=field,survey=survey),\
                             kernel='lumfn',\
                             )
 
-def run_goldqa():
+def run_goldqa(survey):
     print('Running gold QA')
 
     tests = ['zmax_catQA', 'kE_catQA', 'ddp_QA', 'lumfn', 'delta8_qa']
@@ -44,12 +44,13 @@ def run_goldqa():
         try:
             pm.execute_notebook('docs/nb/{}.ipynb'.format(test),\
                                 'test/pm_{}.ipynb'.format(test),\
+                                parameters=dict(survey=survey),\
                                 kernel='lumfn',\
             )
 
-        except Error as E:
+        except:
             print('Failed on {} test with error: '.format(test))
-            print(E)
+            #print(E)
 
     '''
     pm.execute_notebook('docs/nb/kE_catQA.ipynb',\
@@ -76,12 +77,17 @@ def run_goldqa():
     # d8 LF. 
     # desi qa. 
 
-def run_delta8qa():
+def run_delta8qa(survey):
+    fields = fetch_fields(survey)
+
     for field in fields:
         print('Running delta8 QA for field {}'.format(field))
-
-        pm.execute_notebook('docs/nb/delta8_qa.ipynb',\
-                            'test/pm_delta8_qa_{}.ipynb'.format(field),\
-                            parameters=dict(field=field),\
-                            kernel='lumfn',\
-                            )
+
+        pm.execute_notebook('docs/nb/d8LF_qa.ipynb',\
+                    'test/pm_delta8_qa_{}.ipynb'.format(field),\
+                    parameters=dict(field=field, survey=survey),\
+                    kernel='lumfn',\
+                    )
+
+if __name__ == '__main__':
+    test_allnbs()

tidyup.py

@@ -1,26 +1,21 @@
 import os
 import sys
 
-from   findfile import file_check, fields
-
-
-DRYRUN  = os.environ['DRYRUN']
-GOLDDIR = os.environ['GOLD_DIR']
-RANDDIR = os.environ['RANDOMS_DIR']
+from   findfile import file_check, findfile
 
 def tidyup():
     # File check summary. 
-    sys.stdout = open(GOLDDIR + 'summary.log', 'w')
+    fpath      = findfile('summary_log')
+
+    keep       = sys.stdout
+
+    sys.stdout = open(fpath, 'w')
 
     file_check()
 
     sys.stdout.close()
-
-    # Gather Randoms and write to disk. 
-    #
-    # fpaths   = findfile('ddp_n8_d0', dryrun=False, prefix='', field=fields, utier=6)
-    # all_cats = gather_cat(fpaths)
-    # all_cats.pprint()
+
+    sys.stdout = keep
 
 
 if __name__ == '__main__':