V236 release (#4648)

* Fix broken --version option (#4630)

* Fix broken --version option, add CI test, cleanup and simplify related code

* Fix bug preventing .so files from being found

* Fix missing import, typo, more codeclimate

* pycbc_coinc_time needs dqsegdb; deduplicate --help commands

* Added tables to dq results pages (#4633)

* Added tables to dq results pages

* Address Tom's comments and fix bin_trigger_rates_dq

* Fix decimal formatting

* Final change to formatting

* Bugfix: upload prep read psds function (#4644)

* Ensure approximant field is correct format (#4646)

* Bump release number

---------

Co-authored-by: Tito Dal Canton <[email protected]>
Co-authored-by: maxtrevor <[email protected]>
Co-authored-by: Gareth S Cabourn Davies <[email protected]>

bin/all_sky_search/pycbc_bin_trigger_rates_dq

@@ -71,9 +71,9 @@ with HFile(args.trig_file, 'r') as trig_file:
     data_mask = np.zeros(n_triggers_orig, dtype=bool)
     data_mask[idx] = True
 
-    # also get the gated times while we have the file open
+if args.gating_windows:
     gate_times = []
-    if args.gating_windows:
+    with h5.File(args.trig_file, 'r') as trig_file:
         logging.info('Getting gated times')
         try:
             gating_types = trig_file[f'{ifo}/gating'].keys()
@@ -188,6 +188,7 @@ with h5.File(args.output_file, 'w') as f:
             frac_dt = abs(segs) / livetime
             dq_rates[state] = frac_eff / frac_dt
         bin_grp['dq_rates'] = dq_rates
+        bin_grp['num_triggers'] = len(trig_times_bin)
 
     # save dq state segments
     for dq_state, segs in dq_state_segs_dict.items():
@@ -196,7 +197,10 @@ with h5.File(args.output_file, 'w') as f:
         starts, ends = segments_to_start_end(segs)
         dq_grp['segment_starts'] = starts
         dq_grp['segment_ends'] = ends
+        dq_grp['livetime'] = abs(segs)
 
     f.attrs['stat'] = f'{ifo}-dq_stat_info'
+    f.attrs['sngl_ranking'] = args.sngl_ranking
+    f.attrs['sngl_ranking_threshold'] = args.stat_threshold
 
 logging.info('Done!')

bin/all_sky_search/pycbc_prepare_xml_for_gracedb

@@ -96,7 +96,7 @@ def read_psds(psd_files):
         psd = [group["psds"][str(i)] for i in range(len(group["psds"].keys()))]
         psds[ifo] = segmentlist(psd_segment(*segargs) for segargs in zip(
             psd, group["start_time"], group["end_time"]))
-        return psds
+    return psds
 
 psds = read_psds(args.psd_files)
 

bin/plotting/pycbc_page_dq_table

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+""" Make a table of dq state information
+"""
+import sys
+import argparse
+import h5py as h5
+import numpy as np
+
+import pycbc
+import pycbc.results
+from pycbc.version import git_verbose_msg as version
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--version", action="version", version=version)
+parser.add_argument('--ifo', required=True)
+parser.add_argument('--dq-file', required=True)
+parser.add_argument('--verbose', action='count')
+parser.add_argument('--output-file')
+args = parser.parse_args()
+
+dq_states = {
+    'dq_state_0': 'Clean',
+    'dq_state_1': 'DQ Flag',
+    'dq_state_2': 'Autogating',
+}
+
+f = h5.File(args.dq_file, 'r')
+grp = f[args.ifo]['dq_segments']
+
+livetimes = []
+total_livetime = 0
+for dq_state in dq_states:
+    livetime = grp[dq_state]['livetime'][()]
+    livetimes.append(livetime)
+    total_livetime += livetime
+livetimes.append(total_livetime)
+
+frac_livetimes = [lt / total_livetime for lt in livetimes]
+state_names = list(dq_states.values()) + ['Total']
+columns = [state_names, livetimes, frac_livetimes]
+columns = [np.array(c) for c in columns]
+col_names = ['DQ State', 'Livetime', '% of Livetime']
+
+format_strings = [None, '0.0', '0.00%']
+
+html_table = pycbc.results.html_table(columns, col_names,
+                                      page_size=len(state_names),
+                                      format_strings=format_strings)
+title = f'{args.ifo} DQ State Livetimes'
+caption = 'Table of DQ state livetimes'
+
+pycbc.results.save_fig_with_metadata(
+    str(html_table), args.output_file, title=title,
+    caption=caption, cmd=' '.join(sys.argv))

bin/plotting/pycbc_page_template_bin_table

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+""" Make a table of template bin information
+"""
+import sys
+import argparse
+import h5py as h5
+import numpy as np
+
+import pycbc
+import pycbc.results
+from pycbc.version import git_verbose_msg as version
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--version", action="version", version=version)
+parser.add_argument('--ifo', required=True)
+parser.add_argument('--dq-file', required=True)
+parser.add_argument('--verbose', action='count')
+parser.add_argument('--output-file')
+args = parser.parse_args()
+
+f = h5.File(args.dq_file, 'r')
+grp = f[args.ifo]['bins']
+bin_names = list(grp.keys())
+
+sngl_ranking = f.attrs['sngl_ranking']
+sngl_thresh = f.attrs['sngl_ranking_threshold']
+
+livetime = 0
+seg_grp = f[args.ifo]['dq_segments']
+for k in seg_grp.keys():
+    livetime += seg_grp[k]['livetime'][()]
+
+num_templates = []
+num_triggers = []
+total_templates = 0
+total_triggers = 0
+for bin_name in bin_names:
+    bin_grp = grp[bin_name]
+
+    n_tmp = len(bin_grp['tids'][:])
+    num_templates.append(n_tmp)
+    total_templates += n_tmp
+
+    n_trig = bin_grp['num_triggers'][()]
+    num_triggers.append(n_trig)
+    total_triggers += n_trig
+
+bin_names.append('Total')
+num_triggers.append(total_triggers)
+num_templates.append(total_templates)
+frac_triggers = [n / total_triggers for n in num_triggers]
+frac_templates = [n / total_templates for n in num_templates]
+trigger_rate = [n / livetime for n in num_triggers]
+
+col_names = ['Template Bin', 'Number of Templates', '% of Templates',
+             'Number of Loud Triggers', '% of Loud Triggers',
+             'Loud Trigger Rate (Hz)']
+columns = [bin_names, num_templates, frac_templates,
+           num_triggers, frac_triggers, trigger_rate]
+columns = [np.array(c) for c in columns]
+
+format_strings = [None, '#', '0.000%', '#', '0.0%', '0.00E0']
+
+html_table = pycbc.results.html_table(columns, col_names,
+                                      page_size=len(bin_names),
+                                      format_strings=format_strings)
+title = f'{args.ifo} DQ Template Bin Information'
+caption = 'Table of information about template bins ' \
+           + 'used for DQ trigger rate calculations. ' \
+           + 'Loud triggers are defined as those with ' \
+           + f'{sngl_ranking} > {sngl_thresh}.'
+
+pycbc.results.save_fig_with_metadata(
+    str(html_table), args.output_file, title=title,
+    caption=caption, cmd=' '.join(sys.argv))

bin/plotting/pycbc_plot_dq_flag_likelihood

@@ -77,7 +77,10 @@ ax2.set_ylabel('DQ Log Likelihood Penalty')
 ax2.set_ylim(numpy.log(ymin), numpy.log(ymax))
 
 # add meta data and save figure
-plot_title = f'{ifo} DQ Trigger Rates'
+plot_title = f'{ifo}:{args.dq_label} DQ Trigger Rates'
+
+ax.set_title(plot_title)
+
 plot_caption = 'The log likelihood correction \
       during during each dq state for each template bin.'
 pycbc.results.save_fig_with_metadata(

bin/pycbc_convertinjfiletohdf

@@ -150,7 +150,11 @@ class LVKNewStyleInjectionSet(object):
             elif lvk_name == 't_co_gps_add':
                 continue
             else:
-                data[pycbc_name] = self.inj_file[f'{self.subdir}/{lvk_name}'][:]
+                lvk_file_dset = self.inj_file[f'{self.subdir}/{lvk_name}']
+                if lvk_file_dset.dtype.char in ['U', 'O']:
+                    data[pycbc_name] = lvk_file_dset[:].astype('S')
+                else:
+                    data[pycbc_name] = lvk_file_dset[:]
         return data
 
 

bin/workflows/pycbc_make_offline_search_workflow

@@ -561,19 +561,25 @@ for key in final_bg_files:
 
 # DQ log likelihood plots
 for dqf, dql in zip(dqfiles, dqfile_labels):
+    ifo = dqf.ifo
+    outdir = rdir[f'data_quality/{dqf.ifo}_DQ_results']
+
     # plot rates when flag was on
-    outdir = rdir[f'data_quality/{dqf.ifo}_{dql}_trigger_rates']
     wf.make_dq_flag_trigger_rate_plot(workflow, dqf, dql, outdir, tags=[dql])
 
-# DQ background binning plot
-ifo_bins = workflow.cp.get_subsections('bin_templates')
-for ifo in ifo_bins:
+    # make table of dq segment info
+    wf.make_dq_segment_table(workflow, dqf, outdir, tags=[dql])
+
+    # plot background bins
     background_bins = workflow.cp.get_opt_tags(
         'bin_templates', 'background-bins', tags=[ifo])
-    wf.make_template_plot(workflow, hdfbank, rdir['data_quality'],
+    wf.make_template_plot(workflow, hdfbank, outdir,
                           bins=background_bins,
                           tags=[ifo, 'dq_bins'] + bank_tags)
 
+    # make table of template bin info
+    wf.make_template_bin_table(workflow, dqf, outdir, tags=[ifo])
+
 ############################## Setup the injection runs #######################
 
 splitbank_files_inj = wf.setup_splittable_workflow(workflow, [hdfbank],

pycbc/_version.py

@@ -13,70 +13,99 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
 """
 This modules contains a function to provide an argparse action that reports
 extremely verbose version information for PyCBC, lal, and lalsimulation.
 """
 
-import os, sys
+import os
+import sys
+import glob
 import argparse
 import inspect
 import subprocess
 
+
 def print_link(library):
-    err_msg = "Could not execute runtime linker to determine\n" + \
-              "shared library paths for library:\n  " + library + "\n"
-    FNULL = open(os.devnull, 'w')
+    err_msg = (
+        "Could not execute runtime linker to determine\n"
+        f"shared library paths for library:\n  {library}\n"
+    )
     try:
-        link = subprocess.check_output(['ldd', library],
-                                       stderr=FNULL)
+        # Linux
+        link = subprocess.check_output(
+            ['ldd', library],
+            stderr=subprocess.DEVNULL,
+            text=True
+        )
     except OSError:
         try:
-            link = subprocess.check_output(['otool', '-L', library],
-                                           stderr=FNULL)
+            # macOS
+            link = subprocess.check_output(
+                ['otool', '-L', library],
+                stderr=subprocess.DEVNULL,
+                text=True
+            )
         except:
             link = err_msg
     except:
         link = err_msg
     return link
 
 
+def get_lal_info(module, lib_glob):
+    """Return a string reporting the version and runtime library information
+    for a LAL Python import.
+    """
+    module_path = inspect.getfile(module)
+    version_str = (
+        module.git_version.verbose_msg +
+        "\n\nImported from: " + module_path +
+        "\n\nRuntime libraries:\n"
+    )
+    possible_lib_paths = glob.glob(
+        os.path.join(os.path.dirname(module_path), lib_glob)
+    )
+    for lib_path in possible_lib_paths:
+        version_str += print_link(lib_path)
+    return version_str
+
+
 class Version(argparse.Action):
-    """ print the pycbc, lal and lalsimulation versions """
+    """Subclass of argparse.Action that prints version information for PyCBC,
+    LAL and LALSimulation.
+    """
     def __init__(self, nargs=0, **kw):
         super(Version, self).__init__(nargs=nargs, **kw)
 
-
     def __call__(self, parser, namespace, values, option_string=None):
-
         import pycbc
-        version_str="--- PyCBC Version --------------------------\n" + \
-            pycbc.version.git_verbose_msg + \
+
+        version_str = (
+            "--- PyCBC Version --------------------------\n" +
+            pycbc.version.git_verbose_msg +
             "\n\nImported from: " + inspect.getfile(pycbc)
+        )
 
         version_str += "\n\n--- LAL Version ----------------------------\n"
         try:
             import lal.git_version
-            lal_module = inspect.getfile(lal)
-            lal_library = os.path.join( os.path.dirname(lal_module),
-                '_lal.so')
-            version_str += lal.git_version.verbose_msg + \
-            "\n\nImported from: " + lal_module + \
-            "\n\nRuntime libraries:\n" + print_link(lal_library)
         except ImportError:
             version_str += "\nLAL not installed in environment\n"
+        else:
+            version_str += get_lal_info(lal, '_lal*.so')
 
         version_str += "\n\n--- LALSimulation Version-------------------\n"
         try:
             import lalsimulation.git_version
-            lalsimulation_module = inspect.getfile(lalsimulation)
-            lalsimulation_library = os.path.join( os.path.dirname(lalsimulation_module),
-                '_lalsimulation.so')
-            version_str += lalsimulation.git_version.verbose_msg + \
-            "\n\nImported from: " + lalsimulation_module + \
-            "\n\nRuntime libraries:\n" + print_link(lalsimulation_library)
         except ImportError:
             version_str += "\nLALSimulation not installed in environment\n"
+        else:
+            version_str += get_lal_info(lalsimulation, '_lalsimulation*.so')
 
         print(version_str)
         sys.exit(0)
+
+
+__all__ = ['Version']