Make logging better for pycbc_make_offline_workflow (gwastro#4517)

* Make logging better for pycbc_make_offline_workflow

* comment

* Fix bug

* CC complaint - there are some other, but these havent been changed by this PR

* CC

* Update pycbc/workflow/core.py

Co-authored-by: Tito Dal Canton <[email protected]>

* IH/TDC comments

* use warnings module

---------

Co-authored-by: Tito Dal Canton <[email protected]>

bin/workflows/pycbc_make_offline_search_workflow

@@ -57,17 +57,18 @@ def ifo_combos(ifos):
         for ifocomb in combinations:
             yield ifocomb
 
-
-# Log to the screen until we know where the output file is
-logging.basicConfig(format='%(asctime)s:%(levelname)s : %(message)s',
-    level=logging.INFO)
-
 parser = argparse.ArgumentParser(description=__doc__[1:])
 parser.add_argument('--version', action='version', version=__version__)
+parser.add_argument('--verbose', action='count',
+    help="Incrementally add more verbosity")
 wf.add_workflow_command_line_group(parser)
 wf.add_workflow_settings_cli(parser)
 args = parser.parse_args()
 
+# By default, we do logging.info, each --verbose adds a level of verbosity
+logging_level = args.verbose + 1 if args.verbose else logging.INFO
+pycbc.init_logging(logging_level)
+
 container = wf.Workflow(args, args.workflow_name)
 workflow = wf.Workflow(args, args.workflow_name + '-main')
 finalize_workflow = wf.Workflow(args, args.workflow_name + '-finalization')

pycbc/workflow/configuration.py

@@ -28,6 +28,7 @@
 """
 
 import os
+import logging
 import stat
 import shutil
 import subprocess
@@ -87,6 +88,9 @@ def resolve_url(url, directory=None, permissions=None, copy_to_cwd=True):
         # it needs to be available when documentation runs in the CI, and I
         # can't get it to install in the GitHub CI
         import ciecplib
+        # Make the scitokens logger a little quieter
+        # (it is called through ciecpclib)
+        logging.getLogger('scitokens').setLevel(logging.root.level + 10)
         with ciecplib.Session() as s:
             if u.netloc in ("git.ligo.org", "code.pycbc.phy.syr.edu"):
                 # authenticate with git.ligo.org using callback

pycbc/workflow/core.py

@@ -224,8 +224,12 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None,
             # being directly accessible on the execution site.
             # CVMFS is perfect for this! As is singularity.
             self.exe_pfns[exe_site] = exe_path
-        logging.info("Using %s executable "
-                     "at %s on site %s" % (name, exe_url.path, exe_site))
+        logging.debug(
+            "Using %s executable at %s on site %s",
+            name,
+            exe_url.path,
+            exe_site
+        )
 
         # FIXME: This hasn't yet been ported to pegasus5 and won't work.
         #        Pegasus describes two ways to work with containers, and I need

pycbc/workflow/pegasus_workflow.py

@@ -28,8 +28,10 @@
 """
 import os
 import shutil
+import logging
 import tempfile
 import subprocess
+import warnings
 from packaging import version
 from urllib.request import pathname2url
 from urllib.parse import urljoin, urlsplit
@@ -315,6 +317,13 @@ class Workflow(object):
     """
     def __init__(self, name='my_workflow', directory=None, cache_file=None,
                  dax_file_name=None):
+        # Pegasus logging is fairly verbose, quieten it down a bit
+        # This sets the logger to one level less verbose than the root
+        # (pycbc) logger
+
+        # Get the logger associated with the Pegasus workflow import
+        pegasus_logger = logging.getLogger('Pegasus')
+        pegasus_logger.setLevel(logging.root.level + 10)
         self.name = name
         self._rc = dax.ReplicaCatalog()
         self._tc = dax.TransformationCatalog()
@@ -340,7 +349,7 @@ def __init__(self, name='my_workflow', directory=None, cache_file=None,
             self.filename = dax_file_name
         self._adag = dax.Workflow(self.filename)
 
-        # A pegasus job version of this workflow for use if it isncluded
+        # A pegasus job version of this workflow for use if it is included
         # within a larger workflow
         self._as_job = SubWorkflow(self.filename, is_planned=False,
                                    _id=self.name)
@@ -843,10 +852,11 @@ def insert_into_dax(self, rep_cat, sites):
     @classmethod
     def from_path(cls, path):
         """Takes a path and returns a File object with the path as the PFN."""
-        logging.warn("The from_path method in pegasus_workflow is deprecated. "
-                     "Please use File.from_path (for output files) in core.py "
-                     "or resolve_url_to_file in core.py (for input files) "
-                     "instead.")
+        warnings.warn("The from_path method in pegasus_workflow is "
+                      "deprecated. Please use File.from_path (for "
+                      "output files) in core.py or resolve_url_to_file "
+                      "in core.py (for input files) instead.",
+                      DeprecationWarning)
         urlparts = urlsplit(path)
         site = 'nonlocal'
         if (urlparts.scheme == '' or urlparts.scheme == 'file'):