From c0ad2196598b80d7062e70662358ec762aea9f1d Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Fri, 25 Aug 2023 04:27:57 -0700 Subject: [PATCH 1/3] data: update datafind server environment variable --- gwsumm/data/timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gwsumm/data/timeseries.py b/gwsumm/data/timeseries.py index 73106938..9d46a05e 100644 --- a/gwsumm/data/timeseries.py +++ b/gwsumm/data/timeseries.py @@ -553,7 +553,7 @@ def _get_timeseries_dict(channels, segments, config=None, if nds is None and cache is not None: nds = False elif nds is None: - nds = 'LIGO_DATAFIND_SERVER' not in os.environ + nds = 'GWDATAFIND_SERVER' not in os.environ # read new data query &= (abs(new) > 0) From 3b9156afc6054ee42864ec4bda41085be9aac9cd Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Fri, 25 Aug 2023 04:28:30 -0700 Subject: [PATCH 2/3] batch: don't pass getenv=true to condor instead enumerate the variables we want to propagate from the submit environment --- gwsumm/batch.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/gwsumm/batch.py b/gwsumm/batch.py index 499390bc..4d314d0a 100644 --- a/gwsumm/batch.py +++ b/gwsumm/batch.py @@ -42,6 +42,28 @@ __author__ = 'Duncan Macleod ' __credits__ = 'Alex Urban ' +# environment variables to pass to data-processing jobs +GETENV_VARIABLES = [ + # -- IGWN resources + # address of dqsegdb server + "DEFAULT_SEGMENT_SERVER", + # address of gwdatafind server + "GWDATAFIND_SERVER", + # address of NDS2 server + "NDSSERVER", + # -- software stuff + # GWpy customisations + "GWPY*", + # -- Auth handling + # Kerberos + "KRB5*", + # SciTokens + "BEARER_TOKEN*", + "SCITOKEN*", + # X.509 + "X509*", +] + PROG = ('python -m gwsumm.batch' if sys.argv[0].endswith('.py') else os.path.basename(sys.argv[0])) @@ -67,7 +89,6 @@ def __init__(self, universe, tag='gw_summary', logdir, '%s-%s.err' % (tag, self.logtag))) self.set_stdout_file(os.path.join( logdir, '%s-%s.out' % (tag, self.logtag))) - cmds.setdefault('getenv', 'True') for key, val in cmds.items(): if hasattr(self, 'set_%s' % key.lower()): getattr(self, 'set_%s' % key.lower())(val) @@ -524,15 +545,25 @@ def main(args=None): globalconfig = ','.join(args.global_config) jobs = [] + job_kw = { + "subdir": outdir, + "logdir": logdir, + **condorcmds, + } if not args.skip_html_wrapper: htmljob = GWSummaryJob( - 'local', subdir=outdir, logdir=logdir, - tag='%s_local' % args.file_tag, **condorcmds) + 'local', + tag='%s_local' % args.file_tag, + **job_kw, + ) jobs.append(htmljob) if not args.html_wrapper_only: datajob = GWSummaryJob( - universe, subdir=outdir, logdir=logdir, - tag=args.file_tag, **condorcmds) + universe, + tag=args.file_tag, + getenv=" ".join(GETENV_VARIABLES), + **job_kw, + ) jobs.append(datajob) # add common command-line options From bf723986a15918a48d1a3210701b047eec7b36e7 Mon Sep 17 00:00:00 2001 From: Duncan Macleod Date: Mon, 19 Feb 2024 14:49:07 +0000 Subject: [PATCH 3/3] batch: add CONDA_EXE to GETENV_VARIABLES so that it is available in conda to generate the package list --- gwsumm/batch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gwsumm/batch.py b/gwsumm/batch.py index 4d314d0a..f3786df6 100644 --- a/gwsumm/batch.py +++ b/gwsumm/batch.py @@ -52,6 +52,8 @@ # address of NDS2 server "NDSSERVER", # -- software stuff + # Conda (to generate package list for HTML) + "CONDA_EXE", # GWpy customisations "GWPY*", # -- Auth handling