Fix live snr opt and to use with injections without GraceDB uploads (#…

…3023) * Split the data info small frame files * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update pycbc_condition_strain * Assert frame-duration is positive integer * Update pycbc_make_psd_estimation_workflow same variable name `segments` used for function in `ligo` and loop `for ifo, segments in analyzable_segs.items():` producing error. * Update pycbc_condition_strain * Fixed accidental changes to README and psd_estimation * Fixed accidental changes to README and psd_estimation * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Fixed accidental changes to README and psd_estimation * Fixed accidental changes to README and psd_estimation * condition strain removed. will be added later * Split the data info small frame files * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update pycbc_condition_strain * Assert frame-duration is positive integer * Update pycbc_condition_strain * Fixed accidental changes to README and psd_estimation * Fixed accidental changes to README and psd_estimation * Fixed accidental changes to README and psd_estimation * condition strain removed. will be added later * condition_strain as main master * snr-opt should use live_ifos. Also o use snr-opt with injection without GDB. * Added far threshold to snr-opt and moved random_strings to __init__ * Fixed import issues * Make codeclimate happy * fix ifar check to run snr-opt * gid fix * Code climate is harsh! * Code climate... plzzz
gwastro · Dec 2, 2019 · 6a0c463 · 6a0c463
1 parent e58d846
commit 6a0c463
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 28 deletions.
diff --git a/README.md b/README.md
@@ -15,12 +15,12 @@ Documentation is automatically built nightly and available from
 For the detailed installation instructions of PyCBC
 
  * https://pycbc.org/pycbc/latest/html/install.html
-  
+
 Quick Installation
 ```
 pip install lalsuite pycbc
 ```
-  
+
 You can start up an interactive jupyter notebook with PyCBC using either of the links below.
 
 [![Azure Notebooks](https://notebooks.azure.com/launch.png)](https://notebooks.azure.com/nitz/libraries/pycbc)
@@ -41,9 +41,10 @@ A key for the latest release is available at:
 
 [![DOI](https://zenodo.org/badge/31596861.svg)](https://zenodo.org/badge/latestdoi/31596861)
 
-Please see our [citation guidelines](http://pycbc.org/pycbc/latest/html/credit.html) for 
+Please see our [citation guidelines](http://pycbc.org/pycbc/latest/html/credit.html) for
 more details.
 
 [![PyPI version](https://badge.fury.io/py/PyCBC.svg)](https://badge.fury.io/py/PyCBC)
 [![Build Status](https://travis-ci.org/gwastro/pycbc.svg?branch=master)](https://travis-ci.org/gwastro/pycbc)
 [![Research software impact](http://depsy.org/api/package/pypi/PyCBC/badge.svg)](http://depsy.org/package/python/PyCBC)
+
diff --git a/bin/pycbc_live b/bin/pycbc_live
@@ -17,8 +17,6 @@ import itertools
 import platform
 import distutils
 import subprocess
-import random
-import string
 from mpi4py import MPI as mpi
 from pycbc.pool import BroadcastPool
 from pycbc import fft, version, waveform, scheme, makedir
@@ -55,12 +53,6 @@ def combine_ifar_pvalue(ifar, pvalue, livetime):
     # take max of original IFAR and combined IFAR & apply trials factor
     return numpy.maximum(ifar, nifar) / 2.
 
-# https://pynative.com/python-generate-random-string/
-def random_string(stringLength=10):
-    """Generate a random string of fixed length """
-    letters = string.ascii_lowercase
-    return ''.join(random.choice(letters) for i in range(stringLength))
-
 
 class LiveEventManager(object):
     def __init__(self, output_path,
@@ -192,7 +184,7 @@ class LiveEventManager(object):
         for ifo in ifos:
             if data_readers[ifo].near_hwinj():
                 coinc_results['HWINJ'] = True
-                break    
+                break
 
         if 'foreground/ifar' in coinc_results:
             logging.info('computing followup data for coinc')
@@ -222,10 +214,10 @@ class LiveEventManager(object):
 
             end_time = int(coinc_results['foreground/%s/end_time'
                                          % coinc_ifos[0]])
-            fname = 'coinc-{}-{}.xml.gz'.format(end_time, random_string(6))
+            fname = 'coinc-{}-{}.xml.gz'.format(end_time, pycbc.random_string(6))
             fname = os.path.join(self.path, fname)
             logging.info('Coincident candidate! Saving as %s', fname)
-            comments = ['using ranking statistic: %s' 
+            comments = ['using ranking statistic: %s'
                         % args.background_statistic]
 
             ifar = coinc_results['foreground/ifar']
@@ -237,9 +229,9 @@ class LiveEventManager(object):
                 gid = None
                 event.save(fname)
 
-            if self.run_snr_optimization and gid is not None:
+            if self.run_snr_optimization and self.ifar_upload_threshold < ifar:
                 template_id = \
-                    coinc_results['foreground/{}/template_id'.format(ifos[0])]
+                    coinc_results['foreground/{}/template_id'.format(live_ifos[0])]
                 p = props(bank.table[template_id])
                 apr = p.pop('approximant')
                 min_buffer = bank.minimum_buffer + 0.5
@@ -262,7 +254,7 @@ class LiveEventManager(object):
                     curr_data = data_readers[ifo].overwhitened_data(delta_f)
                     curr_data.save(curr_fname)
                     data_fils_str += '{}:{} ' .format(ifo, curr_fname)
-                    curr_fname = fname.replace('.xml.gz', 
+                    curr_fname = fname.replace('.xml.gz',
                                                '_{}_psd.hdf'.format(ifo))
                     curr_psd = curr_data.psd
                     curr_psd.save(curr_fname)
@@ -291,7 +283,9 @@ class LiveEventManager(object):
                     hdfp['template_duration'] = \
                         bank.table[template_id].template_duration
                     hdfp['ifar'] = ifar
-                    hdfp['gid'] = gid
+                    if gid is not None:
+                        hdfp['gid'] = gid
+
                     for ifo in args.channel_name:
                         hdfp['channel_names/{}'.format(ifo)] = \
                             args.channel_name[ifo]
@@ -302,6 +296,9 @@ class LiveEventManager(object):
                 if not self.gracedb_testing:
                     cmd += '--production '
                 cmd += '--verbose '
+                cmd += '--output-path {} '.format(self.path)
+                if self.enable_gracedb_upload:
+                    cmd += '--enable-gracedb-upload '
                 log_fname = \
                         fname.replace('.xml.gz', '_optimize_snr_log.dat')
 
@@ -542,7 +539,7 @@ parser.add_argument('--run-snr-optimization', action='store_true',
                     default=False,
                     help='Run spawned followup processes to maximize SNR for '
                          'any trigger uploaded to GraceDB')
-                   
+
 
 scheme.insert_processing_option_group(parser)
 LiveSingleFarThreshold.insert_args(parser)
@@ -665,12 +662,12 @@ with ctx:
             logging.info('Will calculate %s background', combo)
             estimators.append(Coincer.from_cli(args,
                               len(bank), args.analysis_chunk, list(combo)))
-                              
+
         my_coinc_id = 999999
         def set_coinc_id(i):
             global my_coinc_id
             my_coinc_id = i
-        
+
         def get_coinc(results):
             c = estimators[my_coinc_id]
             r = c.add_singles(results)
@@ -682,7 +679,7 @@ with ctx:
             estim = estimators[my_coinc_id]
             bg_time = estim.background_time / lal.YRJUL_SI
             return estim.ifos, estim.coincs.data, bg_time
-            
+
         coinc_pool = BroadcastPool(len(estimators))
         coinc_pool.allmap(set_coinc_id, range(len(estimators)))
 
@@ -759,7 +756,7 @@ with ctx:
             # Look for coincident triggers and do background estimation
             if args.enable_background_estimation:
                 coinc_results = coinc_pool.broadcast(get_coinc, results)
-                
+
                 # Pick the best coinc in this chunk
                 best_coinc = Coincer.pick_best_coinc(coinc_results)
 

diff --git a/bin/pycbc_optimize_snr b/bin/pycbc_optimize_snr
@@ -127,6 +127,10 @@ parser.add_argument('--gracedb-server', metavar='URL',
                          'If not provided, the default URL is used.')
 parser.add_argument('--production', action='store_true',
                     help='Upload a production event rather than a test event')
+parser.add_argument('--enable-gracedb-upload', action='store_true', default=False,
+                    help='Upload triggers to GraceDB')
+parser.add_argument('--output-path', required=True,
+                    help='Path to a directory to store results in')
 
 args = parser.parse_args()
 pycbc.init_logging(args.verbose)
@@ -150,7 +154,7 @@ for ifo in ifos:
         pass
 
 coinc_ifos = list(coinc_times.keys())
-    
+
 logging.info("Following up GID {}".format(fp['gid'][()]))
 flen = fp['flen'][()]
 approximant = args.approximant
@@ -324,10 +328,16 @@ doc = live.SingleCoincForGraceDB(ifos, coinc_results,
 tmpdir = tempfile.mkdtemp()
 xml_path = os.path.join(
         tmpdir, '{:.3f}.xml.gz'.format(loudest_snr_time_allifos))
-gid = doc.upload(xml_path,
-                 gracedb_server=args.gracedb_server,
-                 testing=(not args.production),
-                 extra_strings=comments)
+if args.enable_gracedb_upload:
+    gid = doc.upload(xml_path,
+                    gracedb_server=args.gracedb_server,
+                    testing=(not args.production),
+                    extra_strings=comments)
+else:
+    gid = None
+    fname = 'coinc-{:.3f}-{}.xml.gz'.format(loudest_snr_time_allifos, pycbc.random_string(6))
+    fname = os.path.join(args.output_path, fname)
+    doc.save(fname)
 
 if gid is not None:
     logging.info('Event uploaded as %s', gid)

diff --git a/pycbc/__init__.py b/pycbc/__init__.py
@@ -31,6 +31,8 @@
 warnings.filterwarnings("ignore", message="numpy.dtype size changed")
 warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
 import logging
+import random
+import string
 
 try:
     # This will fail when pycbc is imported during the build process,
@@ -127,3 +129,9 @@ def makedir(path):
     HAVE_OMP = False
 else:
     HAVE_OMP = True
+
+# https://pynative.com/python-generate-random-string/
+def random_string(stringLength=10):
+    """Generate a random string of fixed length """
+    letters = string.ascii_lowercase
+    return ''.join(random.choice(letters) for i in range(stringLength))