diff --git a/.travis.yml b/.travis.yml index 0e6dab2..28b0ccf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,60 @@ -language: python -dist: xenial +language: minimal -python: - - '3.5' - - '3.6' - - '3.7' +os: linux + +env: + global: + - COVERAGE_STORAGE="json" + - CONDA_PKGS_DIRS="${HOME}/.cache/conda/pkgs" + +matrix: + fast_finish: true + + include: + # conda builds + - name: "conda:3.6" + env: PYTHON_VERSION="3.6" + - name: "conda:3.7" + env: PYTHON_VERSION="3.7" + - name: "conda:3.8" + env: PYTHON_VERSION="3.8" + - name: "conda:3.9" + env: PYTHON_VERSION="3.9" before_install: - - python -m pip install -q --upgrade pip - - python -m pip install -r requirements.txt + - curl -LO https://raw.githubusercontent.com/gwpy/gwpy/master/ci/parse-conda-requirements.py + - curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + - bash miniconda.sh -b -p ${HOME}/miniconda + - source "${HOME}/miniconda/etc/profile.d/conda.sh" + - conda config --set always_yes yes --set changeps1 no + - conda config --add channels conda-forge + - travis_retry conda update --quiet --yes conda + # Useful for debugging any issues with conda + - conda info --all install: + # create a conda environment + - travis_retry conda create --quiet --yes --name gwvetci python=${PYTHON_VERSION} pip setuptools + - travis_retry conda activate gwvetci + - travis_retry python ./parse-conda-requirements.py requirements.txt -o conda-reqs.txt + - travis_retry conda install --quiet --yes --update-all --name gwvetci --file conda-reqs.txt + # clean up + - rm -f conda-reqs.txt parse-conda-requirements.py + # install this version - python -m pip install . script: # run flake8 - - python -m flake8 gwvet/**/*.py - - python -m flake8 bin/* + - python -m flake8 . # test executables - - python -m coverage run --append $(which gwvet) --help - - python -m coverage run --append $(which gwvet-hug) --help - - python -m coverage run --append $(which gwvet-vdf) --help + - python -m coverage run --append --source gwvet -m gwvet --help + - python -m coverage run --append --source gwvet -m gwvet.hug --help + - python -m coverage run --append --source gwvet -m gwvet.vdf --help -cache: - pip: true before_cache: + - travis_retry conda clean --quiet --yes --all - rm -f $HOME/.cache/pip/log/debug.log +cache: + pip: true + directories: + - ${HOME}/.cache/conda/pkgs diff --git a/bin/gwvet b/bin/gwvet deleted file mode 100644 index 33899a9..0000000 --- a/bin/gwvet +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# Copyright (C) Duncan Macleod (2013) -# -# This file is part of GWpy VET. -# -# GWpy VET is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# GWSumm is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with GWpy VET. If not, see . - -"""Study the performance of one or more data-quality flags. - -This utility evalutes the performance of a given flag, or set of flags, -based on a number of pre-defined metrics. - -All of the command-line options can be given through the configuration files -with the named sections (given in the section headings in --help), while any -arguments given explicitly on the command-line take precedence. -""" - -import argparse -import os.path - -from gwpy.time import to_gps - -from gwsumm import globalv -from gwsumm.config import GWSummConfigParser -from gwsumm.utils import vprint - -from gwvet import __version__ -from gwvet.metric import get_metric -from gwvet.cli import ACTIONS - -__author__ = 'Duncan Macleod ' - - -# ----------------------------------------------------------------------------- -# read command-line - -class ParseGPS(argparse.Action): - """Parse arbitrary input into GPS format from command-line - """ - def __call__(self, parser, namespace, values, option_string=None): - values = float(values) - setattr(namespace, self.dest, to_gps(values)) - - -prog = os.path.basename(__file__) - -# create the top-level parser -parser = argparse.ArgumentParser( - description=__doc__, epilog="All questions and comments should be " - "addressed to detchar@ligo.org.") -parser._optionals.title = 'Optional arguments' -parser.add_argument('-V', '--version', action='version', version=__version__) - -sharedopts = argparse.ArgumentParser(add_help=False) -sharedopts.add_argument('-v', '--verbose', action='store_true', default=False, - help="print verbose progress to stdout, " - "default: %(default)s") - -genopts = sharedopts.add_argument_group( - 'General options', - 'Give standard parameters that define the scope of this study.') -genopts.add_argument('-s', '--gps-start-time', action=ParseGPS, - help='GPS start time for study.', required=True) -genopts.add_argument('-e', '--gps-end-time', action=ParseGPS, - help='GPS end time for study.', required=True) -genopts.add_argument('-m', '--metric', action='append', dest='metrics', - metavar='METRIC', - help='metric to use in study, can be given ' - 'multiple times') - -trigopts = sharedopts.add_argument_group( - "Event trigger options", - "Configure analysis triggers used to evaluate performance of data-quality " - "flags on a search-specific basis.") -trigopts.add_argument('-c', '--channel', action='store', type=str, - help="name of primary event trigger channel.") -trigopts.add_argument('-f', '--trigger-format', - help='format of trigger files, if given') -trigsource = trigopts.add_mutually_exclusive_group(required=False) -trigsource.add_argument('-t', '--auto-locate-triggers', action='store_true', - default=False, - help="use trigfind to auto-locate trigger files, " - "default: %(default)s.") -trigsource.add_argument('-T', '--trigger-file', action='append', type=str, - help="path to trigger file. This option " - "can be given multiple times.") - -segopts = sharedopts.add_argument_group( - "Segment database options") -segopts.add_argument('-u', '--segment-url', - default='https://segdb.ligo.caltech.edu', - help='URL of segment database, default: %(default)s') - -outopts = sharedopts.add_argument_group( - 'Output options', - 'Configure HTML and figure output') -outopts.add_argument('-o', '--html-dir', default=None, - help='target directory for output HTML, ' - 'default: %(default)s') -outopts.add_argument('-l', '--label', default='Vetoes', - help='text label for plots and HTML, ' - 'default: %(default)s') - -subparsers = parser.add_subparsers( - dest='mode', title='Select one of the following study modes', - description='[run %s --help for detailed help]' % prog) -subparser = {} - -# ----------------------------------------------------------------------------- -# Single-flag mode - -for action, mod in ACTIONS.items(): - subparser[action] = mod.add_command_line_arguments( - subparsers, [sharedopts]) - -# ----------------------------------------------------------------------------- -# parse and execute - -args = parser.parse_args() - -# set default metrics -if not args.metrics and (args.auto_locate_triggers or args.trigger_file): - args.metrics = ['deadtime', 'efficiency'] -elif not args.metrics: - args.metrics = ['deadtime'] -args.metrics = map(get_metric, args.metrics) - -globalv.VERBOSE = args.verbose - -# construct config -config = GWSummConfigParser() -config.add_section('general') -config.set('general', 'gps-start-time', str(int(args.gps_start_time))) -config.set('general', 'gps-end-time', str(int(args.gps_end_time))) -config.add_section('segment-database') -config.set('segment-database', 'url', args.segment_url) - -# set output directory -if args.html_dir: - if not os.path.isdir(args.html_dir): - os.makedirs(args.html_dir) - os.chdir(args.html_dir) - -action = args.mode -ACTIONS[action].run(args, config) - -vprint(""" ------------------------------------------------------------------------------- -All done. Thank you. ------------------------------------------------------------------------------- -""") diff --git a/bin/gwvet-hug b/bin/gwvet-hug deleted file mode 100644 index 98377c0..0000000 --- a/bin/gwvet-hug +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# Copyright (C) Duncan Macleod (2013) -# -# This file is part of GWpy VET. -# -# GWpy VET is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# GWSumm is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with GWpy VET. If not, see . - -import argparse -import datetime -import glob -import os -try: - from configparser import ConfigParser -except ImportError: # python < 3 - from ConfigParser import RawConfigParser as ConfigParser - -import numpy - -from gwpy.segments import DataQualityFlag -from gwpy.time import tconvert -from gwpy.segments import SegmentList, Segment - -from gwvet import __version__ - -__author__ = 'Erika Cowan ' - - -# command line parsing -parser = argparse.ArgumentParser( - description='dq_seg_finder.py is a program that grabs triggers ' - 'and segments for hveto,UPVh, and OVL for any time period, and ' - 'concatenates them into one segment file, and one trigger file. ' - 'It then creates a DQ Flag for the given type and time period, ' - 'spits out a .xml file, and generates the .ini file needed to ' - 'run VET. For questions or concerns, contact Erika Cowan at ' - 'erika.cowan@ligo.org') -parser.add_argument('-V', '--version', action='version', version=__version__) -parser.add_argument('gps_start_time', type=int, - help='GPS start time') -parser.add_argument('gps_end_time', type=int, - help='GPS end time') -parser.add_argument('directory_path', type=str, - help='Directory path for triggers and segments') -parser.add_argument('type_dq_flag', type=str, - help='Either hveto, UPVh, OVL') -parser.add_argument('-a', '--hveto_analysis_seg', type=str, - help='Offline hveto O1 offline analysis segment, one of ' - '4,5,6,8,9') -parser.add_argument('-o', '--online_offline', type=str, - help='Selector for offline or online, this is for hveto') -args = parser.parse_args() - -# algorithm that loads in triggers from file for any time segment - - -def grab_time_triggers(glob_wildcard): - time_segs = SegmentList([]) - start_time_utc = tconvert(args.gps_start_time) - for filename in glob.glob(glob_wildcard): - data = SegmentList.read(filename) - print('grabbing trigger file:' + filename) - start_end_seg = Segment(args.gps_start_time, args.gps_end_time) - c = data & SegmentList([start_end_seg]) - time_segs += c - start_time_utc += datetime.timedelta(days=1) - return time_segs - - -def grab_time_segments(glob_wildcard): - known_start = [] - known_end = [] - start_time_utc = tconvert(args.gps_start_time) - for filename in glob.glob(glob_wildcard): - if os.path.isfile(filename): - segments = numpy.atleast_2d(numpy.loadtxt(filename, delimiter=',')) - known_start = [segments[i, 0] for i in range(len(segments))] - known_end = [segments[i, 1] for i in range(len(segments))] - start_time_utc += datetime.timedelta(days=1) - - for index in range(len(known_start)): - g.write(str(known_start[index]) + " " + str(known_end[index]) + "\n") - - -# algorithm that takes segment/trigger list and writes to file -def write_segs(trig_seg_list, output_file): - total_triggers = trig_seg_list.coalesce() - total_triggers.write(output_file) - - -# check to make sure we're within the time window of aLIGO, -# and that end_time is after start_time -if args.gps_start_time < 971574400: # roughly the end of S6 - parser.error("gps_start_time before S6") -if args.gps_end_time < args.gps_start_time: - parser.error("end_time is before gps_start_time") - -# finds beginning of day for given gps time -start_of_day = tconvert(args.gps_start_time) -start_of_day_utc = start_of_day.replace(hour=0, minute=0, second=0) -start_of_day_gps = tconvert(start_of_day) - -# finds UTC version of start/end times -start_time_utc = tconvert(args.gps_start_time) -end_time_utc = tconvert(args.gps_end_time) - -# opens files to be ready for writing -f = open("total_" + args.type_dq_flag + "_trigs.txt", "w") # all triggers -g = open("total_" + args.type_dq_flag + "_segs.txt", "w") # all segments - -# choosing to read in hveto -if args.type_dq_flag == 'hveto': - - print('Data Quality Flag chosen is hveto, stored in the path ' - '%s' % args.directory_path) - - # choosing the offline hveto option for O1, runs by Josh Smith - if args.online_offline == 'offline': - analysis_segs_45689 = ['4', '5', '6', '7', '9'] - analysis_segs_237 = ['2', '3'] - if args.hveto_analysis_seg in analysis_segs_45689: - pattern_trigs_hveto = os.path.join( - args.directory_path, 'analysis%s' % args.hveto_analysis_seg, - 'H1-omicron_BOTH-*-DARM', '*VETO_SEGS_ROUND*.txt') - pattern_segs_hveto = os.path.join( - args.directory_path, 'analysis%s' % args.hveto_analysis_seg, - 'H1-omicron_BOTH-*-DARM', 'segs.txt') - - elif args.hveto_analysis_seg in analysis_segs_237: - pattern_trigs_hveto = os.path.join( - args.directory_path, 'H1-omicron_BOTH-*-DARM', - '*VETO_SEGS_ROUND*.txt') - pattern_segs_hveto = os.path.join( - args.directory_path, 'H1-omicron_BOTH-*-DARM', 'segs.txt') - - elif args.hveto_analysis_seg == '8': - pattern_trigs_hveto = os.path.join( - args.directory_path, '*VETO_SEGS_ROUND*.txt') - pattern_segs_hveto = os.path.join( - args.directory_path, 'segs.txt') - else: - raise ValueError('Must choose from O1 analysis segments ' - '1 through 9') - print('Data Quality Flag chosen is hveto, stored in the path ' - '%s' % args.directory_path) - - while start_time_utc < end_time_utc: - day = start_time_utc.day - month = start_time_utc.month - year = start_time_utc.year - - triggers = grab_time_triggers(pattern_trigs_hveto) - - # Ideally we would be able to use the same algorithm, but - # SegmentList.read doesn't support csv, which is the format - # that segment files are recorded in. So, we want to - # temporarily use another method to read in segments. - segments = grab_time_segments(pattern_segs_hveto) - - start_time_utc += datetime.timedelta(days=1) - - write_segs(triggers, f) - # segments.write(g) - - elif args.online_offline == 'online': - - # These paths are currently hardwired for online searches. - pattern_trigs_hveto = os.path.join( - args.directory_path, '{}{:02}', '{}{:02}{:02}', - '*86400-DARM', '*VETO_SEGS_ROUND*.txt') - pattern_segs_hveto = os.path.join( - args.directory_path, '{}{:02}', '{}{:02}{:02}', - '*86400-DARM', 'segs.txt') - - triggers = SegmentList([]) - segments = SegmentList([]) - - while start_time_utc < end_time_utc: - day = start_time_utc.day - month = start_time_utc.month - year = start_time_utc.year - wildcard_trigs_hveto = pattern_trigs_hveto.format( - year, month, year, month, day) - wildcard_segs_hveto = pattern_segs_hveto.format( - year, month, year, month, day) - triggers = grab_time_triggers(wildcard_trigs_hveto) - - # Ideally we would be able to use the same algorithm, but - # SegmentList.read doesn't support csv, which is the format - # segment files are recorded in. So, we want to temporarily - # use another method to read segments in. - segments = grab_time_segments(wildcard_segs_hveto) - - start_time_utc += datetime.timedelta(days=1) - - write_segs(triggers, f) - - # segments.write(g) - - else: - print('Did not choose online or offline. Please choose.') - -# choosing to read in UPVh! -elif args.type_dq_flag == 'UPVh': - - print('Data-quality flag chosen is %s, stored in the path %s' % ( - args.type_dq_flag, args.directory_path)) - - pattern_trigs_UPVh = os.path.join( - args.directory_path, 'DARM_LOCK_{}_{}-H', 'H1:*veto.txt') - pattern_segs_UPVh = os.path.join( - args.directory_path, 'DARM_LOCK_{}_{}-H', 'segments.txt') - triggers = SegmentList([]) - segments = SegmentList([]) - while start_of_day_utc < end_time_utc: - start_of_day_gps = tconvert(start_of_day_utc) - nextday_utc = start_of_day_utc + datetime.timedelta(days=1) - nextday_gps = tconvert(nextday_utc) - wildcard_UPVh_trigs = pattern_trigs_UPVh.format( - start_of_day_gps, nextday_gps) - wildcard_UPVh_segs = pattern_segs_UPVh.format( - start_of_day_gps, nextday_gps) - triggers = grab_time_triggers(wildcard_UPVh_trigs) - segments = grab_time_triggers(wildcard_UPVh_segs) - start_of_day_utc += datetime.timedelta(days=1) - write_segs(triggers, f) - write_segs(segments, g) - -else: # forgot to choose UPVh or hveto - raise ValueError('Did not give a valid data-quality tool, please choose ' - 'from hveto, UPVh, or OVL.') -f.close() -g.close() - -# creating DQ .xml file - -# construct flag and filename -flag_name = 'H1:' + args.type_dq_flag + '-RND:1' -name = 'segments_' + args.type_dq_flag + '_RND.xml' - -# reading in segment files -try: - knownsegments = numpy.loadtxt('total_' + args.type_dq_flag + '_segs.txt') -except OSError: - print("No total_{}_segs.txt file in current working directory. " - "It should have been produced from last loop. " - "If this file is empty, that may mean you have no active segments " - "during this time period.".format(args.type_dq_flag)) - -known_start = [knownsegments[i, 0] for i in range(len(knownsegments))] -known_end = [knownsegments[i, 1] for i in range(len(knownsegments))] - -# reading in trigger files -data = numpy.loadtxt('total_' + args.type_dq_flag + '_trigs.txt') - -# get an array for the start_time and end_time of each segment -start_time = [data[i, 1] for i in range(len(data))] -end_time = [data[i, 2] for i in range(len(data))] - -# create a data quality flag object -flag = DataQualityFlag( - flag_name, active=zip(start_time, end_time), - known=zip(known_start, known_end)) - -# write flag -flag.write(name) - -print("Created DQ Flag: " + flag_name + " in .xml form as: " + name) - -# creating VET .ini file - -config = ConfigParser() - -config.add_section('plugins') -config.set('plugins', 'gwvet.tabs', ' ') - -config.add_section('states') -config.set('states', 'Science', '%(ifo)s:DMT-ANALYSIS_READY:1') - -config.add_section('segment-database') -config.set('segment-database', 'url', 'https://segments.ligo.org') - -config.add_section('') -config.set('', 'type', 'veto-flag') -config.set('', 'event-channel', '%(ifo)s:GDS-CALIB_STRAIN') -config.set('', 'event-generator', 'Omicron') -config.set('', 'metrics', - "'Deadtime',\n'Efficiency', \n'Efficiency/Deadtime', " - "\n'Efficiency | SNR>=8', \n'Efficiency/Deadtime | SNR>=8', " - "\n'Efficiency | SNR>=20', \n'Efficiency/Deadtime | SNR>=20', " - "\n'Efficiency | SNR>=100', \n'Efficiency/Deadtime | SNR>=100', " - "\n'Use percentage', \n'Loudest event by SNR'") - -config.add_section('tab-SNR-6') -config.set('tab-SNR-6', 'name', 'SNR 6') -config.set('tab-SNR-6', 'type', 'veto-flag') -config.set('tab-SNR-6', 'shortname', 'SNR 6') -config.set('tab-SNR-6', 'flags', flag_name) -config.set('tab-SNR-6', 'states', "Science") -config.set('tab-SNR-6', 'segmentfile', name) - -with open(args.type_dq_flag + '_segs.ini', 'wb') as configfile: - config.write(configfile) - -print('\n Created %s_segs.ini. You have everything you need to run VET now! ' - '\n' % args.type_dq_flag) -print('To run VET,first go into %s_segs.ini, and delete the line that only ' - 'contains [], then save and exit the .ini file.\n' % args.type_dq_flag) -print('Finally, run the command: \n' - '$ gw_summary gps %s %s -f /home/detchar/etc/summary/configurations/' - 'defaults.ini -f %s_segs.ini' % ( - args.gps_start_time, args.gps_end_time, args.type_dq_flag)) diff --git a/bin/gwvet-vdf b/bin/gwvet-vdf deleted file mode 100644 index 97ef628..0000000 --- a/bin/gwvet-vdf +++ /dev/null @@ -1,438 +0,0 @@ -#!/usr/bin/env python - -"""Analyse a veto definer file in full using the GWpy VET package -""" - -import os -import sys -import re -from math import (ceil, floor) -from argparse import ArgumentParser -from getpass import getuser -from subprocess import Popen -try: - from urllib.parse import urlparse - from urllib.request import urlopen -except ImportError: # python < 3 - from urllib2 import urlopen - from urlparse import urlparse -try: - from configparser import NoSectionError -except ImportError: # python < 3 or no backports.configparser - from ConfigParser import NoSectionError - -from gwpy.time import to_gps -from gwpy.segments import (Segment, DataQualityFlag, DataQualityDict) - -from gwsumm.config import GWSummConfigParser as ConfigParser -from gwsumm.state import SummaryState -from gwsumm.utils import (re_cchar, mkdir, get_default_ifo) - -from gwvet import __version__ - -__author__ = 'Duncan Macleod ' - -try: - IFO = get_default_ifo() -except ValueError: - IFO = None -DEFAULT_METRICS = ['Deadtime', 'Efficiency', 'Efficiency/Deadtime', - 'Use percentage', 'Loudest event by SNR'] - - -def add_config_section(config, section, **params): - config.add_section(section) - for key, val in params.items(): - config.set(section, key, val) - - -def configure_veto_tab(section, parent, state, flags, segmentfile, metrics, - **params): - tab = 'tab-%s' % section - config.add_section(tab) - params.setdefault('type', 'veto-flag') - params.setdefault('name', section) - if parent is not None: - params.setdefault('parent', parent) - params.setdefault('flags', ','.join(flags)) - params.setdefault('union', '|'.join(flags)) - params.setdefault('intersection', '&'.join(flags)) - params.setdefault('states', state.key) - params.setdefault('veto-name', params['name']) - params.setdefault('metrics', ','.join(metrics)) - # set others - if 'event-channel' in params: - params.setdefault('before', '%(event-channel)s') - params.setdefault('after', '%(event-channel)s#%(union)s') - params.setdefault('vetoed', '%(event-channel)s@%(union)s') - for key, val in params.items(): - config.set(tab, key, val) - return tab - - -# ----------------------------------------------------------------------------- -# -# set up command line and parse -# -# ----------------------------------------------------------------------------- - -parser = ArgumentParser(description=__doc__) -parser.add_argument('-v', '--verbose', action='store_true', - help='print verbose output') -parser.add_argument('-V', '--version', action='version', version=__version__) -parser._positionals.title = 'Positional arguments' -parser._optionals.title = 'Optional arguments' - -# required argument -parser.add_argument('veto-definer-file', help='path to veto definer file') -parser.add_argument('gps-start-time', type=to_gps, - help='GPS start time/date of analysis') -parser.add_argument('gps-end-time', type=to_gps, - help='GPS end time/date of analysis') - -analargs = parser.add_argument_group('Analysis options') -analargs.add_argument('-f', '--config-file', type=os.path.abspath, - default=[], action='append', - help='path to INI file defining this analysis') -analargs.add_argument('-i', '--ifo', default=IFO, - help='prefix of IFO to study, default: %(default)s') -analargs.add_argument('-o', '--output-directory', default=os.curdir, - type=os.path.abspath, - help='output directory path, default: %(default)s, ' - 'this path should be web-viewable') -analargs.add_argument('-c', '--categories', default='1,2,3,4', - help='list of categories to analyse, ' - 'default: %(default)s') -analargs.add_argument('-m', '--metric', action='append', - help='name of metric to use in analysis, ' - 'can be given multiple times, default: %s' - % DEFAULT_METRICS) -analargs.add_argument('-I', '--independent', action='store_true', - help='analyse categories independently, rather than ' - 'hierarchichally, default: hierarchichally') -analargs.add_argument('-g', '--global-config', action='append', default=[], - help='path to gwsumm configuration file passed to all ' - 'gw_summary jobs') - -trigargs = parser.add_argument_group('Trigger options') -trigargs.add_argument('-x', '--event-channel', - default=IFO and '%s:GDS-CALIB_STRAIN' % IFO or None, - help='name of event trigger channel, ' - 'default: %(default)s') -trigargs.add_argument('-G', '--event-generator', default='Omicron', - help='name of event trigger generator, ' - 'default: %(default)s') -trigargs.add_argument('-X', '--event-file', help='path to event cache file') - -segargs = parser.add_argument_group('Segment options') -segargs.add_argument( - '-a', '--analysis-segments', action='append', - default=IFO and ['%s:DMT-ANALYSIS_READY:1' % IFO] or None, - help='flag indicating analysis time, or path of segment file containing ' - 'segments for a single flag, default: %(default)s, can be given ' - 'multiple times to use the intersection of many flags, or the union ' - 'of many files') -segargs.add_argument('-n', '--analysis-name', default='Analysis', - help='Human-readable name for summary state, ' - 'e.g. \'Science\', default: %(default)s') -segargs.add_argument('-t', '--segment-url', dest='segdb', - default='https://segments.ligo.org', - help='url of segment database, default: %(default)s') -segargs.add_argument('-S', '--on-segdb-error', default='raise', - choices=['raise', 'ignore', 'warn'], - help='how to handle (dq)segdb errors, ' - 'default: %(default)s') - -args = parser.parse_args() - -# parse command line options -ifo = args.ifo -if not args.ifo: - parser.error('--ifo must be given if not obvious from the host') -start = getattr(args, 'gps-start-time') -end = getattr(args, 'gps-end-time') -span = Segment(start, end) -duration = int(ceil(end) - floor(start)) -categories = args.categories.split(',') -for i, c in enumerate(categories): - try: - categories[i] = int(c) - except (TypeError, ValueError): - pass -vetofile = getattr(args, 'veto-definer-file') -if not urlparse(vetofile).netloc: - vetofile = os.path.abspath(vetofile) - -if not args.metric: - args.metric = DEFAULT_METRICS - - -# ----------------------------------------------------------------------------- -# -# set things up -# -# ----------------------------------------------------------------------------- - -tag = '%d-%d' % (start.seconds, end.seconds) -outdir = os.path.abspath(os.path.join(args.output_directory, tag)) -mkdir(outdir) -os.chdir(outdir) -mkdir('etc', 'segments', 'condor') - -# ----------------------------------------------------------------------------- -# -# segment handling -# -# ----------------------------------------------------------------------------- - -os.chdir('segments') -ALLSEGMENTS = DataQualityDict() - -# -- get analysis segments ---------------------- - -aflags = args.analysis_segments -asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo) -for i, flag in enumerate(aflags): - # use union of segments from a file - if os.path.isfile(flag): - asegments += DataQualityFlag.read(flag) - # or intersection of segments from multiple flags - else: - new = DataQualityFlag.query(flag, start, end, url=args.segdb) - if i: - asegments.known &= new.known - asegments.active &= new.active - else: - asegments.known = new.known - asegments.active = new.active -ALLSEGMENTS[asegments.name] = asegments - -if os.path.isfile(aflags[0]): - asegments.filename = aflags - -# -- read veto definer and process -------------- - -if urlparse(vetofile).netloc: - tmp = urlopen(vetofile) - vetofile = os.path.abspath(os.path.basename(vetofile)) - with open(vetofile, 'w') as f: - f.write(tmp.read()) - print('Downloaded veto definer file') -vdf = DataQualityDict.from_veto_definer_file( - vetofile, format='ligolw', start=start, end=end, ifo=ifo) -print('Read %d flags from veto definer' % len(vdf.keys())) - -# populate veto definer file from database -vdf.populate(source=args.segdb, on_error=args.on_segdb_error) -ALLSEGMENTS += vdf - -# organise flags into categories -flags = dict((c, DataQualityDict()) for c in categories) -for name, flag in vdf.items(): - try: - flags[flag.category][name] = flag - except KeyError: - pass - -# find the states and segments for each category -states, after, oldtitle = (dict(), None, '') -for i, category in enumerate(categories): - title = isinstance(category, int) and 'Cat %d' % category or category - tag = re_cchar.sub('_', str(title).upper()) - states[category] = SummaryState( - 'After %s' % oldtitle, - key=tag, - known=after.known, - active=after.active, - definition=after.name, - ) if i else SummaryState( - args.analysis_name, - key=args.analysis_name, - definition=asegments.name, - ) - try: - segs = flags[category].union() - except TypeError: # no flags - segs = DataQualityFlag() - segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag) - ALLSEGMENTS[segs.name] = segs - after = (after - segs) if i else (asegments - segs) - after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag) - ALLSEGMENTS[after.name] = after - oldtitle = title - -# write all segments to disk -segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' - % (ifo, start.seconds, duration)) -ALLSEGMENTS.write(segfile) - -os.chdir(os.pardir) - -if args.verbose: - print("All segments accessed and written to\n%s" % segfile) - -# ----------------------------------------------------------------------------- -# -# job preparation -# -# ----------------------------------------------------------------------------- - -os.chdir('etc') - -configs = [] -for category in categories: - title = isinstance(category, int) and 'Category %d' % category or category - tab = 'tab-%s' % title - - config = ConfigParser() - - # add segment-database configuration - add_config_section(config, 'segment-database', url=args.segdb) - - # add plot configurations - pconfig = ConfigParser() - pconfig.read(args.config_file) - for section in pconfig.sections(): - if section.startswith('plot-'): - config._sections[section] = pconfig._sections[section].copy() - - try: - plots = pconfig.items('plots-%s' % category, raw=True) - except NoSectionError: - try: - plots = pconfig.items('plots', raw=True) - except NoSectionError: - plots = [] - - # add state - if args.independent: - state = states[categories[0]] - else: - state = states[category] - sname = 'state-%s' % state.key - add_config_section(config, sname, key=state.key, name=state.name, - definition=state.definition, filename=segfile) - - # add plugin - add_config_section(config, 'plugins', **{'gwvet.tabs': ''}) - - # define metrics - if category == 1: - metrics = ['Deadtime'] - else: - metrics = args.metric - - # define summary tab - if category == 1: - tab = configure_veto_tab( - title, title, state, flags[category].keys(), segfile, - metrics, name='Summary', **{'veto-name': title}) - else: - tab = configure_veto_tab( - title, title, state, flags[category].keys(), segfile, - metrics, name='Summary', **{ - 'veto-name': title, - 'event-channel': args.event_channel, - 'event-generator': args.event_generator, - }) - if len(categories) == 1: - config.set(tab, 'index', - '%(gps-start-time)s-%(gps-end-time)s/index.html') - for key, value in plots: - if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605 - config.set(tab, key, '%%(union)s,%s' % value) - if '%s-labels' % key not in plots: - config.set(tab, '%s-labels' % key, 'Union,%(flags)s') - else: - config.set(tab, key, value) - - # now a tab for each flag - for flag in flags[category]: - if category == 1: - tab = configure_veto_tab( - flag, title, state, [flag], segfile, metrics) - else: - tab = configure_veto_tab( - flag, title, state, [flag], segfile, metrics, **{ - 'event-channel': args.event_channel, - 'event-generator': args.event_generator}) - if args.event_file: - config.set(tab, 'event-file', args.event_file) - for key, value in plots: - config.set(tab, key, value) - - if len(categories) > 1 and category != categories[-1]: - with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: - config.write(f) - configs.append(os.path.abspath(f.name)) - -# configure summary job -if len(categories) > 1: - state = states[categories[0]] - add_config_section(config, 'state-%s' % state.key, key=state.key, - name=state.name, definition=state.definition, - filename=segfile) - try: - plots = pconfig.items('plots', raw=True) - except NoSectionError: - plots = [] - flags = [f for c in categories for f in flags[c].keys()] - tab = configure_veto_tab( - 'Impact of full veto definer file', None, state, flags, - segfile, args.metric, shortname='Summary', - index='%(gps-start-time)s-%(gps-end-time)s/index.html', - **{'event-channel': args.event_channel, - 'event-generator': args.event_generator, - 'veto-name': 'All vetoes'}) - if args.event_file: - config.set(tab, 'event-file', args.event_file) - for key, value in plots: - config.set(tab, key, value) - with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: - config.write(f) - configs.append(os.path.abspath(f.name)) - -os.chdir(os.pardir) - -if args.verbose: - print("Generated configuration files for each category") - -# ----------------------------------------------------------------------------- -# -# condor preparation -# -# ----------------------------------------------------------------------------- - -os.chdir(os.pardir) - -# get condor variables -if getuser() == 'detchar': - accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy' -else: - accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy' - -cmd = ['gw_summary_pipe', - '--gps-start-time', str(start.seconds), - '--gps-end-time', str(end.seconds), - '--ifo', ifo, - '--file-tag', 'gwpy-vet', - '--condor-command', 'accounting_group=%s' % accgroup, - '--condor-command', 'accounting_group_user=%s' % getuser(), - '--on-segdb-error', args.on_segdb_error, - '--output-dir', args.output_directory, - ] -for cf in args.global_config: - cmd.extend(('--global-config', cf)) -for cf in configs: - cmd.extend(('--config-file', cf)) -if args.verbose: - cmd.append('--verbose') - -if args.verbose: - print('Generating summary DAG via:\n') - print(' '.join(cmd)) - print('') - -proc = Popen(cmd) -proc.communicate() -sys.exit(proc.returncode) diff --git a/gwvet/__main__.py b/gwvet/__main__.py new file mode 100644 index 0000000..dc82cb3 --- /dev/null +++ b/gwvet/__main__.py @@ -0,0 +1,249 @@ +# coding=utf-8 +# Copyright (C) Duncan Macleod (2013) +# +# This file is part of GWpy VET. +# +# GWpy VET is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GWSumm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GWpy VET. If not, see . + +"""Study the performance of one or more data-quality flags. + +This utility evalutes the performance of a given flag, or set of flags, +based on a number of pre-defined metrics. + +All of the command-line options can be given through the configuration files +with the named sections (given in the section headings in --help), while any +arguments given explicitly on the command-line take precedence. +""" + +import argparse +import os.path +import sys + +from gwpy.time import to_gps + +from gwdetchar.cli import logger + +from gwsumm import globalv +from gwsumm.config import GWSummConfigParser + +from gwvet import __version__ +from gwvet.metric import get_metric +from gwvet.cli import ACTIONS + +__author__ = 'Duncan Macleod ' +__credits__ = 'Alex Urban ' + +PROG = ('python -m gwvet' if sys.argv[0].endswith('.py') + else os.path.basename(sys.argv[0])) +LOGGER = logger(name=PROG.split('python -m ').pop()) + + +# -- utilities ---------------------------------------------------------------- + +class ParseGPS(argparse.Action): + """Parse arbitrary input into GPS format from command-line + """ + def __call__(self, parser, namespace, values, option_string=None): + values = float(values) + setattr(namespace, self.dest, to_gps(values)) + + +# -- parse command-line ------------------------------------------------------- + +def create_parser(): + """Create a command-line parser for this entry point + """ + # initialize the argument parser + parser = argparse.ArgumentParser( + prog=PROG, + description=__doc__, + epilog="All questions and comments should be " + "addressed to detchar@ligo.org.", + ) + + # optional arguments + parser._optionals.title = 'Optional arguments' + parser.add_argument( + '-V', + '--version', + action='version', + version=__version__, + ) + sharedopts = argparse.ArgumentParser(add_help=False) + sharedopts.add_argument( + '-v', + '--verbose', + action='store_true', + default=False, + help="print verbose progress to stdout, default: %(default)s", + ) + + # general arguments + genopts = sharedopts.add_argument_group( + 'General options', + 'Standard parameters that define the scope of this study', + ) + genopts.add_argument( + '-s', + '--gps-start-time', + action=ParseGPS, + help='GPS start time for study', + required=True, + ) + genopts.add_argument( + '-e', + '--gps-end-time', + action=ParseGPS, + help='GPS end time for study', + required=True, + ) + genopts.add_argument( + '-m', + '--metric', + action='append', + dest='metrics', + metavar='METRIC', + help='metric to use in study, can be given multiple times', + ) + + # trigger arguments + trigopts = sharedopts.add_argument_group( + "Event trigger options", + "Configure analysis triggers used to evaluate performance " + "of data-quality flags on a search-specific basis", + ) + trigopts.add_argument( + '-c', + '--channel', + action='store', + type=str, + help="name of primary event trigger channel", + ) + trigopts.add_argument( + '-f', + '--trigger-format', + help='format of trigger files, if given', + ) + trigsource = trigopts.add_mutually_exclusive_group(required=False) + trigsource.add_argument( + '-t', + '--auto-locate-triggers', + action='store_true', + default=False, + help="use trigfind to auto-locate trigger files, default: %(default)s", + ) + trigsource.add_argument( + '-T', + '--trigger-file', + action='append', + type=str, + help="path to trigger file, can be given multiple times", + ) + + # segdb options + segopts = sharedopts.add_argument_group( + "Segment database options", + ) + segopts.add_argument( + '-u', + '--segment-url', + default='https://segdb.ligo.caltech.edu', + help='URL of segment database, default: %(default)s', + ) + + # output options + outopts = sharedopts.add_argument_group( + 'Output options', + 'Configure HTML and figure output', + ) + outopts.add_argument( + '-o', + '--html-dir', + default=None, + help='target directory for output HTML, default: %(default)s', + ) + outopts.add_argument( + '-l', + '--label', + default='Vetoes', + help='text label for plots and HTML, default: %(default)s', + ) + + # toggle single-flag analysis mode + subparser = {} + subparsers = parser.add_subparsers( + dest='mode', + title='Select one of the following study modes', + description='[run {} --help for detailed help]'.format(PROG), + ) + for (action, mod) in ACTIONS.items(): + subparser[action] = mod.add_command_line_arguments( + subparsers, [sharedopts]) + + # return the argument parser + return parser + + +# -- main code block ---------------------------------------------------------- + +def main(args=None): + """Run the VET command-line interface + """ + parser = create_parser() + args = parser.parse_args(args=args) + + # -- setup -------------------------------------- + + LOGGER.info(" -- GW Veto Evaluation and Testing system -- ") + + # set default metrics + if ( + not args.metrics + and (args.auto_locate_triggers or args.trigger_file) + ): + args.metrics = ['deadtime', 'efficiency'] + elif not args.metrics: + args.metrics = ['deadtime'] + args.metrics = map(get_metric, args.metrics) + + # set verbosity level + globalv.VERBOSE = args.verbose + + # construct config + LOGGER.info("Loading configuration") + config = GWSummConfigParser() + config.add_section('general') + config.set('general', 'gps-start-time', str(int(args.gps_start_time))) + config.set('general', 'gps-end-time', str(int(args.gps_end_time))) + config.add_section('segment-database') + config.set('segment-database', 'url', args.segment_url) + + # set output directory + if args.html_dir: + if not os.path.isdir(args.html_dir): + os.makedirs(args.html_dir) + os.chdir(args.html_dir) + + # -- execute mode action ------------------------ + + action = args.mode + ACTIONS[action].run(args, config) + + LOGGER.info(" -- Data products written, all done -- ") + + +# -- run from command-line ---------------------------------------------------- + +if __name__ == "__main__": + main() diff --git a/gwvet/hug.py b/gwvet/hug.py new file mode 100644 index 0000000..1bc8041 --- /dev/null +++ b/gwvet/hug.py @@ -0,0 +1,403 @@ +# coding=utf-8 +# Copyright (C) Duncan Macleod (2013) +# +# This file is part of GWpy VET. +# +# GWpy VET is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GWSumm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GWpy VET. If not, see . + +"""Command-line tool that grabs triggers and segments for hveto, UPV, and +OVL for any time period, then concatenates them into one segment file and +one trigger file. This tool then produces a DQ flag for the given type and +period, spits out a .xml file, and generates the .ini file needed to run VET. +""" + +import argparse +import datetime +import glob +import os +import sys + +from configparser import ConfigParser + +import numpy + +from gwpy.segments import DataQualityFlag +from gwpy.segments import SegmentList, Segment +from gwpy.time import (to_gps, tconvert) + +from gwdetchar.cli import logger + +from gwvet import __version__ + +__author__ = 'Erika Cowan ' +__credits__ = 'Alex Urban ' + +PROG = ('python -m gwvet.hug' if sys.argv[0].endswith('.py') + else os.path.basename(sys.argv[0])) +LOGGER = logger(name=PROG.split('python -m ').pop()) + + +def grab_time_triggers(wildcard, start, end): + """Retrieve triggers from a given GPS time range + """ + time_segs = SegmentList([]) + start_time_utc = tconvert(start) + for filename in glob.glob(wildcard): + data = SegmentList.read(filename) + LOGGER.info(' '.join(['grabbing trigger file:', filename])) + start_end_seg = Segment(start, end) + c = data & SegmentList([start_end_seg]) + time_segs += c + start_time_utc += datetime.timedelta(days=1) + return time_segs + + +def grab_time_segments(wildcard, start, segfile): + """Retrieve time segments from a given start time + """ + known_start = [] + known_end = [] + start_time_utc = tconvert(start) + for filename in glob.glob(wildcard): + if os.path.isfile(filename): + segments = numpy.atleast_2d(numpy.loadtxt(filename, delimiter=',')) + known_start = [segments[i, 0] for i in range(len(segments))] + known_end = [segments[i, 1] for i in range(len(segments))] + start_time_utc += datetime.timedelta(days=1) + for index in range(len(known_start)): + segfile.write( + str(known_start[index]) + " " + str(known_end[index]) + "\n" + ) + + +def write_segs(trig_seg_list, output_file): + """Write a list of trigger segments to an output file + """ + total_triggers = trig_seg_list.coalesce() + total_triggers.write(output_file) + + +# -- parse command-line ------------------------------------------------------- + +def create_parser(): + """Create a command-line parser for this entry point + """ + # initialize argument parser + parser = argparse.ArgumentParser( + prog=PROG, + description=__doc__, + ) + + # positional arguments + parser.add_argument( + 'gpsstart', + type=to_gps, + help='GPS start time', + ) + parser.add_argument( + 'gpsend', + type=to_gps, + help='GPS end time', + ) + parser.add_argument( + 'directory_path', + type=str, + help='Path to output directory for triggers and segments', + ) + parser.add_argument( + 'dq_flag_type', + type=str, + help='DQ flag type, either hveto, UPVh, OVL', + ) + + # optional flags + parser.add_argument( + '-V', + '--version', + action='version', + version=__version__, + ) + parser.add_argument( + '-a', + '--hveto_analysis_seg', + type=str, + help='Hveto O1 offline analysis segment, one of 4,5,6,8,9', + ) + parser.add_argument( + '-o', + '--online_offline', + type=str, + help='Selector for offline or online, needed only for hveto', + ) + + # return the argument parser + return parser + + +# -- main code block ---------------------------------------------------------- + +def main(args=None): + """Run the gwvet-hug CLI tool + """ + parser = create_parser() + args = parser.parse_args(args=args) + + # check to make sure we're within the time window of aLIGO, + # and that end_time is after start_time + if args.gpsstart < 971574400: # roughly the end of S6 + parser.error("gpsstart before S6") + if args.gpsend < args.gpsstart: + parser.error("end_time is before gpsstart") + + # finds beginning of day for given gps time + start_of_day = tconvert(args.gpsstart) + start_of_day_utc = start_of_day.replace(hour=0, minute=0, second=0) + start_of_day_gps = tconvert(start_of_day) + + # finds UTC version of start/end times + start_time_utc = tconvert(args.gpsstart) + end_time_utc = tconvert(args.gpsend) + + # opens files to be ready for writing + f = open("total_" + args.dq_flag_type + "_trigs.txt", "w") # all triggers + g = open("total_" + args.dq_flag_type + "_segs.txt", "w") # all segments + + # choosing to read in hveto + if args.dq_flag_type == 'hveto': + + LOGGER.info( + 'Data Quality Flag chosen is hveto, stored in the path ' + '%s' % args.directory_path) + + # choosing the offline hveto option for O1, runs by Josh Smith + if args.online_offline == 'offline': + analysis_segs_45689 = ['4', '5', '6', '7', '9'] + analysis_segs_237 = ['2', '3'] + if args.hveto_analysis_seg in analysis_segs_45689: + pattern_trigs_hveto = os.path.join( + args.directory_path, + 'analysis%s' % args.hveto_analysis_seg, + 'H1-omicron_BOTH-*-DARM', + '*VETO_SEGS_ROUND*.txt', + ) + pattern_segs_hveto = os.path.join( + args.directory_path, + 'analysis%s' % args.hveto_analysis_seg, + 'H1-omicron_BOTH-*-DARM', + 'segs.txt', + ) + + elif args.hveto_analysis_seg in analysis_segs_237: + pattern_trigs_hveto = os.path.join( + args.directory_path, 'H1-omicron_BOTH-*-DARM', + '*VETO_SEGS_ROUND*.txt') + pattern_segs_hveto = os.path.join( + args.directory_path, 'H1-omicron_BOTH-*-DARM', 'segs.txt') + + elif args.hveto_analysis_seg == '8': + pattern_trigs_hveto = os.path.join( + args.directory_path, '*VETO_SEGS_ROUND*.txt') + pattern_segs_hveto = os.path.join( + args.directory_path, 'segs.txt') + else: + raise ValueError('Must choose from O1 analysis segments ' + '1 through 9') + LOGGER.info( + 'Data Quality Flag chosen is hveto, stored in the path ' + '%s' % args.directory_path) + + while start_time_utc < end_time_utc: + day = start_time_utc.day + month = start_time_utc.month + year = start_time_utc.year + + triggers = grab_time_triggers( + pattern_trigs_hveto, args.gpsstart, args.gpsend) + + # Ideally we would be able to use the same algorithm, but + # SegmentList.read doesn't support csv, which is the format + # that segment files are recorded in. So, we want to + # temporarily use another method to read in segments. + segments = grab_time_segments( + pattern_segs_hveto, args.gpsstart, g) + + start_time_utc += datetime.timedelta(days=1) + + write_segs(triggers, f) + + elif args.online_offline == 'online': + + # These paths are currently hardwired for online searches. + pattern_trigs_hveto = os.path.join( + args.directory_path, '{}{:02}', '{}{:02}{:02}', + '*86400-DARM', '*VETO_SEGS_ROUND*.txt') + pattern_segs_hveto = os.path.join( + args.directory_path, '{}{:02}', '{}{:02}{:02}', + '*86400-DARM', 'segs.txt') + + triggers = SegmentList([]) + segments = SegmentList([]) + + while start_time_utc < end_time_utc: + day = start_time_utc.day + month = start_time_utc.month + year = start_time_utc.year + wildcard_trigs_hveto = pattern_trigs_hveto.format( + year, month, year, month, day) + wildcard_segs_hveto = pattern_segs_hveto.format( + year, month, year, month, day) + triggers = grab_time_triggers( + wildcard_trigs_hveto, args.gpsstart, args.gpsend) + + # Ideally we would be able to use the same algorithm, but + # SegmentList.read doesn't support csv, which is the format + # segment files are recorded in. So, we want to temporarily + # use another method to read segments in. + segments = grab_time_segments( + wildcard_segs_hveto, args.gpsstart, g) + + start_time_utc += datetime.timedelta(days=1) + + write_segs(triggers, f) + + # segments.write(g) + + else: + LOGGER.info('Did not choose online or offline. Please choose.') + + # choosing to read in UPVh! + elif args.dq_flag_type == 'UPVh': + + LOGGER.info('Data-quality flag chosen is %s, stored in the path %s' % ( + args.dq_flag_type, args.directory_path)) + + pattern_trigs_UPVh = os.path.join( + args.directory_path, 'DARM_LOCK_{}_{}-H', 'H1:*veto.txt') + pattern_segs_UPVh = os.path.join( + args.directory_path, 'DARM_LOCK_{}_{}-H', 'segments.txt') + triggers = SegmentList([]) + segments = SegmentList([]) + while start_of_day_utc < end_time_utc: + start_of_day_gps = tconvert(start_of_day_utc) + nextday_utc = start_of_day_utc + datetime.timedelta(days=1) + nextday_gps = tconvert(nextday_utc) + wildcard_UPVh_trigs = pattern_trigs_UPVh.format( + start_of_day_gps, nextday_gps) + wildcard_UPVh_segs = pattern_segs_UPVh.format( + start_of_day_gps, nextday_gps) + triggers = grab_time_triggers( + wildcard_UPVh_trigs, args.gpsstart, args.gpsend) + segments = grab_time_triggers( + wildcard_UPVh_segs, args.gpsstart, args.gpsend) + start_of_day_utc += datetime.timedelta(days=1) + write_segs(triggers, f) + write_segs(segments, g) + + else: # forgot to choose UPVh or hveto + raise ValueError( + 'Did not give a valid data-quality tool, ' + 'please choose from hveto, UPVh, or OVL' + ) + f.close() + g.close() + + # creating DQ .xml file + + # construct flag and filename + flag_name = 'H1:' + args.dq_flag_type + '-RND:1' + name = 'segments_' + args.dq_flag_type + '_RND.xml' + + # reading in segment files + try: + knownsegments = numpy.loadtxt( + 'total_' + args.dq_flag_type + '_segs.txt') + except OSError: + LOGGER.info( + "No total_{}_segs.txt file in current working directory. " + "It should have been produced from last loop. " + "If this file is empty, that may mean you have no active " + "segments during this time period.".format(args.dq_flag_type)) + + known_start = [knownsegments[i, 0] for i in range(len(knownsegments))] + known_end = [knownsegments[i, 1] for i in range(len(knownsegments))] + + # reading in trigger files + data = numpy.loadtxt('total_' + args.dq_flag_type + '_trigs.txt') + + # get an array for the start_time and end_time of each segment + start_time = [data[i, 1] for i in range(len(data))] + end_time = [data[i, 2] for i in range(len(data))] + + # create a data quality flag object + flag = DataQualityFlag( + flag_name, active=zip(start_time, end_time), + known=zip(known_start, known_end)) + + # write flag + flag.write(name) + + LOGGER.info("Created DQ Flag: " + flag_name + " in .xml form as: " + name) + + # creating VET .ini file + + config = ConfigParser() + + config.add_section('plugins') + config.set('plugins', 'gwvet.tabs', ' ') + + config.add_section('states') + config.set('states', 'Science', '%(ifo)s:DMT-ANALYSIS_READY:1') + + config.add_section('segment-database') + config.set('segment-database', 'url', 'https://segments.ligo.org') + + config.add_section('') + config.set('', 'type', 'veto-flag') + config.set('', 'event-channel', '%(ifo)s:GDS-CALIB_STRAIN') + config.set('', 'event-generator', 'Omicron') + config.set('', 'metrics', + "'Deadtime',\n'Efficiency', \n'Efficiency/Deadtime', " + "\n'Efficiency | SNR>=8', \n'Efficiency/Deadtime | SNR>=8', " + "\n'Efficiency | SNR>=20', \n'Efficiency/Deadtime | SNR>=20', " + "\n'Efficiency | SNR>=100', \n'Efficiency/Deadtime | SNR>=100'," + " \n'Use percentage', \n'Loudest event by SNR'") + + config.add_section('tab-SNR-6') + config.set('tab-SNR-6', 'name', 'SNR 6') + config.set('tab-SNR-6', 'type', 'veto-flag') + config.set('tab-SNR-6', 'shortname', 'SNR 6') + config.set('tab-SNR-6', 'flags', flag_name) + config.set('tab-SNR-6', 'states', "Science") + config.set('tab-SNR-6', 'segmentfile', name) + + with open(args.dq_flag_type + '_segs.ini', 'wb') as configfile: + config.write(configfile) + + LOGGER.info( + '\n Created %s_segs.ini. You have everything you need to run VET now! ' + '\n' % args.dq_flag_type) + LOGGER.info( + 'To run VET,first go into %s_segs.ini, and delete the line that only ' + 'contains [], then save and exit the .ini file.\n' % args.dq_flag_type) + LOGGER.info( + 'Finally, run the command: \n' + '$ gw_summary gps %s %s -f /home/detchar/etc/summary/configurations/' + 'defaults.ini -f %s_segs.ini' % ( + args.gpsstart, args.gpsend, args.dq_flag_type)) + + +# -- run from command-line ---------------------------------------------------- + +if __name__ == "__main__": + main() diff --git a/gwvet/metric/metrics.py b/gwvet/metric/metrics.py index b734eb8..204d2b6 100644 --- a/gwvet/metric/metrics.py +++ b/gwvet/metric/metrics.py @@ -210,6 +210,7 @@ def safety(segments, injections, threshold=SAFETY_THRESHOLD): def loudest_event_metric_factory(column): column = column.lower() + @_use_dqflag def loudest_event(segments, before, after=None): """Percentage reduction in the amplitude of the loudest event by %s diff --git a/gwvet/vdf.py b/gwvet/vdf.py new file mode 100644 index 0000000..e23806a --- /dev/null +++ b/gwvet/vdf.py @@ -0,0 +1,542 @@ +# coding=utf-8 +# Copyright (C) Duncan Macleod (2013) +# +# This file is part of GWpy VET. +# +# GWpy VET is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GWSumm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GWpy VET. If not, see . + +"""Analyse a veto definer file in full using the GWpy VET package +""" + +import argparse +import os +import re +import sys + +from configparser import NoSectionError +from getpass import getuser +from math import (ceil, floor) +from urllib.parse import urlparse +from urllib.request import urlopen + +from gwpy.segments import ( + DataQualityFlag, + DataQualityDict, +) +from gwpy.time import to_gps + +from gwdetchar.cli import logger + +from gwsumm import batch +from gwsumm.config import GWSummConfigParser as ConfigParser +from gwsumm.state import SummaryState +from gwsumm.utils import ( + get_default_ifo, + re_cchar, + mkdir, +) + +from gwvet import __version__ + +__author__ = 'Duncan Macleod ' +__credits__ = 'Alex Urban ' + +try: + IFO = get_default_ifo() +except ValueError: + IFO = None + +DEFAULT_METRICS = [ + 'Deadtime', + 'Efficiency', + 'Efficiency/Deadtime', + 'Use percentage', + 'Loudest event by SNR', +] + +PROG = ('python -m gwvet.vdf' if sys.argv[0].endswith('.py') + else os.path.basename(sys.argv[0])) +LOGGER = logger(name=PROG.split('python -m ').pop()) + + +# -- utilities ---------------------------------------------------------------- + +def add_config_section(config, section, **params): + """Add a section to the global configuration + """ + config.add_section(section) + for key, val in params.items(): + config.set(section, key, val) + + +def configure_veto_tab(config, section, parent, state, flags, + segmentfile, metrics, **params): + """Configure the tab corresponding to a given veto + """ + tab = 'tab-%s' % section + config.add_section(tab) + params.setdefault('type', 'veto-flag') + params.setdefault('name', section) + if parent is not None: + params.setdefault('parent', parent) + params.setdefault('flags', ','.join(flags)) + params.setdefault('union', '|'.join(flags)) + params.setdefault('intersection', '&'.join(flags)) + params.setdefault('states', state.key) + params.setdefault('veto-name', params['name']) + params.setdefault('metrics', ','.join(metrics)) + # set others + if 'event-channel' in params: + params.setdefault('before', '%(event-channel)s') + params.setdefault('after', '%(event-channel)s#%(union)s') + params.setdefault('vetoed', '%(event-channel)s@%(union)s') + for key, val in params.items(): + config.set(tab, key, val) + return tab + + +# -- parse command-line ------------------------------------------------------- + +def create_parser(): + """Create a command-line parser for this entry point + """ + # initialize argument parser + parser = argparse.ArgumentParser( + prog=PROG, + description=__doc__, + ) + parser.add_argument( + '-v', + '--verbose', + action='store_true', + help='print verbose output', + ) + parser.add_argument( + '-V', + '--version', + action='version', + version=__version__, + ) + parser._positionals.title = 'Positional arguments' + parser._optionals.title = 'Optional arguments' + + # required arguments + parser.add_argument( + 'veto-definer-file', + help='path to veto definer file', + ) + parser.add_argument( + 'gpsstart', + type=to_gps, + help='GPS start time/date of analysis', + ) + parser.add_argument( + 'gpsend', + type=to_gps, + help='GPS end time/date of analysis', + ) + + # analysis options + analargs = parser.add_argument_group('Analysis options') + analargs.add_argument( + '-f', + '--config-file', + type=os.path.abspath, + default=[], + action='append', + help='path to INI file defining this analysis', + ) + analargs.add_argument( + '-i', + '--ifo', + default=IFO, + help='prefix of IFO to study, default: %(default)s', + ) + analargs.add_argument( + '-o', + '--output-directory', + default=os.curdir, + type=os.path.abspath, + help='output directory path, default: %(default)s, ' + 'this path should be web-viewable', + ) + analargs.add_argument( + '-c', + '--categories', + default='1,2,3,4', + help='list of categories to analyse, default: %(default)s', + ) + analargs.add_argument( + '-m', + '--metric', + action='append', + help='name of metric to use in analysis, can be given ' + 'multiple times, default: %s' % DEFAULT_METRICS, + ) + analargs.add_argument( + '-I', + '--independent', + action='store_true', + help='analyse categories independently, rather than ' + 'hierarchichally, default: hierarchichally', + ) + analargs.add_argument( + '-g', + '--global-config', + action='append', + default=[], + help='path to gwsumm configuration file passed to all gw_summary jobs', + ) + + # trigger options + trigargs = parser.add_argument_group('Trigger options') + trigargs.add_argument( + '-x', + '--event-channel', + default=IFO and '%s:GDS-CALIB_STRAIN' % IFO or None, + help='name of event trigger channel, default: %(default)s', + ) + trigargs.add_argument( + '-G', + '--event-generator', + default='Omicron', + help='name of event trigger generator, default: %(default)s', + ) + trigargs.add_argument( + '-X', + '--event-file', + help='path to event cache file', + ) + + # segment options + segargs = parser.add_argument_group('Segment options') + segargs.add_argument( + '-a', + '--analysis-segments', + action='append', + default=IFO and ['%s:DMT-ANALYSIS_READY:1' % IFO] or None, + help='flag indicating analysis time, or path of segment file ' + 'containing segments for a single flag, default: %(default)s, ' + 'can be given multiple times to use the intersection of many ' + 'flags, or the union of many files', + ) + segargs.add_argument( + '-n', + '--analysis-name', + default='Analysis', + help='Human-readable name for summary state, ' + 'e.g. \'Science\', default: %(default)s', + ) + segargs.add_argument( + '-t', + '--segment-url', + dest='segdb', + default='https://segments.ligo.org', + help='url of segment database, default: %(default)s', + ) + segargs.add_argument( + '-S', + '--on-segdb-error', + default='raise', + choices=['raise', 'ignore', 'warn'], + help='how to handle (dq)segdb errors, default: %(default)s', + ) + + # return the argument parser + return parser + + +# -- main code block ---------------------------------------------------------- + +def main(args=None): + """Run the online Guardian node visualization tool + """ + parser = create_parser() + args = parser.parse_args(args=args) + + # parse command line options + ifo = args.ifo + if not args.ifo: + parser.error('--ifo must be given if not obvious from the host') + start = getattr(args, 'gpsstart') + end = getattr(args, 'gpsend') + duration = int(ceil(end) - floor(start)) + categories = args.categories.split(',') + for i, c in enumerate(categories): + try: + categories[i] = int(c) + except (TypeError, ValueError): + pass + vetofile = getattr(args, 'veto-definer-file') + vetofile = (urlparse(vetofile).netloc + or os.path.abspath(vetofile)) + args.metric = args.metric or DEFAULT_METRICS + + # -- setup -------------------------------------- + + tag = '%d-%d' % (start.seconds, end.seconds) + outdir = os.path.abspath(os.path.join(args.output_directory, tag)) + mkdir(outdir) + os.chdir(outdir) + mkdir('etc', 'segments', 'condor') + + # -- segment handling --------------------------- + + os.chdir('segments') + ALLSEGMENTS = DataQualityDict() + + # -- get analysis segments ---------------------- + + aflags = args.analysis_segments + asegments = DataQualityFlag('%s:VET-ANALYSIS_SEGMENTS:0' % ifo) + for i, flag in enumerate(aflags): + # use union of segments from a file + if os.path.isfile(flag): + asegments += DataQualityFlag.read(flag) + # or intersection of segments from multiple flags + else: + new = DataQualityFlag.query(flag, start, end, url=args.segdb) + if i: + asegments.known &= new.known + asegments.active &= new.active + else: + asegments.known = new.known + asegments.active = new.active + ALLSEGMENTS[asegments.name] = asegments + + if os.path.isfile(aflags[0]): + asegments.filename = aflags + + # -- read veto definer and process -------------- + + if urlparse(vetofile).netloc: + tmp = urlopen(vetofile) + vetofile = os.path.abspath(os.path.basename(vetofile)) + with open(vetofile, 'w') as f: + f.write(tmp.read()) + LOGGER.info('Downloaded veto definer file') + vdf = DataQualityDict.from_veto_definer_file( + vetofile, format='ligolw', start=start, end=end, ifo=ifo) + LOGGER.info('Read %d flags from veto definer' % len(vdf.keys())) + + # populate veto definer file from database + vdf.populate(source=args.segdb, on_error=args.on_segdb_error) + ALLSEGMENTS += vdf + + # organise flags into categories + flags = dict((c, DataQualityDict()) for c in categories) + for name, flag in vdf.items(): + try: + flags[flag.category][name] = flag + except KeyError: + pass + + # find the states and segments for each category + states, after, oldtitle = (dict(), None, '') + for i, category in enumerate(categories): + title = isinstance(category, int) and 'Cat %d' % category or category + tag = re_cchar.sub('_', str(title).upper()) + states[category] = SummaryState( + 'After %s' % oldtitle, + key=tag, + known=after.known, + active=after.active, + definition=after.name, + ) if i else SummaryState( + args.analysis_name, + key=args.analysis_name, + definition=asegments.name, + ) + try: + segs = flags[category].union() + except TypeError: # no flags + segs = DataQualityFlag() + segs.name = '%s:VET-ANALYSIS_%s:0' % (ifo, tag) + ALLSEGMENTS[segs.name] = segs + after = (after - segs) if i else (asegments - segs) + after.name = '%s:VET-ANALYSIS_AFTER_%s:0' % (ifo, tag) + ALLSEGMENTS[after.name] = after + oldtitle = title + + # write all segments to disk + segfile = os.path.abspath('%s-VET_SEGMENTS-%d-%d.xml.gz' + % (ifo, start.seconds, duration)) + ALLSEGMENTS.write(segfile) + + os.chdir(os.pardir) + + if args.verbose: + LOGGER.debug("All segments accessed and written to\n%s" % segfile) + + # -- job preparation ---------------------------- + + os.chdir('etc') + + configs = [] + for category in categories: + title = (isinstance(category, int) + and 'Category %d' % category or category) + tab = 'tab-%s' % title + config = ConfigParser() + + # add segment-database configuration + add_config_section(config, 'segment-database', url=args.segdb) + + # add plot configurations + pconfig = ConfigParser() + pconfig.read(args.config_file) + for section in pconfig.sections(): + if section.startswith('plot-'): + config._sections[section] = pconfig._sections[section].copy() + + try: + plots = pconfig.items('plots-%s' % category, raw=True) + except NoSectionError: + try: + plots = pconfig.items('plots', raw=True) + except NoSectionError: + plots = [] + + # add state + if args.independent: + state = states[categories[0]] + else: + state = states[category] + sname = 'state-%s' % state.key + add_config_section(config, sname, key=state.key, name=state.name, + definition=state.definition, filename=segfile) + + # add plugin + add_config_section(config, 'plugins', **{'gwvet.tabs': ''}) + + # define metrics + if category == 1: + metrics = ['Deadtime'] + else: + metrics = args.metric + + # define summary tab + if category == 1: + tab = configure_veto_tab( + config, title, title, state, flags[category].keys(), segfile, + metrics, name='Summary', **{'veto-name': title}) + else: + tab = configure_veto_tab( + config, title, title, state, flags[category].keys(), segfile, + metrics, name='Summary', **{ + 'veto-name': title, + 'event-channel': args.event_channel, + 'event-generator': args.event_generator, + }) + if len(categories) == 1: + config.set(tab, 'index', + '%(gps-start-time)s-%(gps-end-time)s/index.html') + for key, value in plots: + if re.match('%\(flags\)s (?:plot-)?segments', value): # noqa: W605 + config.set(tab, key, '%%(union)s,%s' % value) + if '%s-labels' % key not in plots: + config.set(tab, '%s-labels' % key, 'Union,%(flags)s') + else: + config.set(tab, key, value) + + # now a tab for each flag + for flag in flags[category]: + if category == 1: + tab = configure_veto_tab( + config, flag, title, state, [flag], segfile, metrics) + else: + tab = configure_veto_tab( + config, flag, title, state, [flag], segfile, metrics, **{ + 'event-channel': args.event_channel, + 'event-generator': args.event_generator}) + if args.event_file: + config.set(tab, 'event-file', args.event_file) + for key, value in plots: + config.set(tab, key, value) + + if len(categories) > 1 and category != categories[-1]: + with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: + config.write(f) + configs.append(os.path.abspath(f.name)) + + # configure summary job + if len(categories) > 1: + state = states[categories[0]] + add_config_section(config, 'state-%s' % state.key, key=state.key, + name=state.name, definition=state.definition, + filename=segfile) + try: + plots = pconfig.items('plots', raw=True) + except NoSectionError: + plots = [] + flags = [f for c in categories for f in flags[c].keys()] + tab = configure_veto_tab( + config, 'Impact of full veto definer file', None, state, flags, + segfile, args.metric, shortname='Summary', + index='%(gps-start-time)s-%(gps-end-time)s/index.html', + **{'event-channel': args.event_channel, + 'event-generator': args.event_generator, + 'veto-name': 'All vetoes'}) + if args.event_file: + config.set(tab, 'event-file', args.event_file) + for key, value in plots: + config.set(tab, key, value) + with open('%s.ini' % re_cchar.sub('-', title.lower()), 'w') as f: + config.write(f) + configs.append(os.path.abspath(f.name)) + + os.chdir(os.pardir) + + if args.verbose: + LOGGER.debug("Generated configuration files for each category") + + # -- condor preparation ------------------------- + + os.chdir(os.pardir) + + # get condor variables + if getuser() == 'detchar': + accgroup = 'ligo.prod.o1.detchar.dqproduct.gwpy' + else: + accgroup = 'ligo.dev.o1.detchar.dqproduct.gwpy' + + gwsumm_args = [ + '--gps-start-time', str(start.seconds), + '--gps-end-time', str(end.seconds), + '--ifo', ifo, + '--file-tag', 'gwpy-vet', + '--condor-command', 'accounting_group=%s' % accgroup, + '--condor-command', 'accounting_group_user=%s' % getuser(), + '--on-segdb-error', args.on_segdb_error, + '--output-dir', args.output_directory, + ] + for cf in args.global_config: + gwsumm_args.extend(('--global-config', cf)) + for cf in configs: + gwsumm_args.extend(('--config-file', cf)) + if args.verbose: + gwsumm_args.append('--verbose') + + if args.verbose: + LOGGER.debug('Generating summary DAG via:\n') + LOGGER.debug(' '.join([batch.PROG] + gwsumm_args)) + + # execute gwsumm in batch mode + batch.main(args=gwsumm_args) + + +# -- run from command-line ---------------------------------------------------- + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index f8a955b..63a7518 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,21 +2,20 @@ setuptools # core requirements -astropy>=1.0 +astropy>=3.0.0 decorator dqsegdb -gwdetchar>=1.0.0 -gwpy>=1.0.0 -gwsumm>=1.0.1 +gwdetchar>=2.0.0 +gwpy>=2.0.0 +gwsumm>=2.0.0 gwtrigfind lscsoft-glue>=1.60.0 MarkupPy -matplotlib>=1.5 -numpy>=1.7 +matplotlib>=3.1 +numpy>=1.16 scipy # testing extras -pytest>=2.8,<3.7 -pytest-cov -coverage flake8 +pytest>=3.3.0 +pytest-cov>=2.4.0 diff --git a/setup.cfg b/setup.cfg index 3d75bf1..6f524c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,6 @@ [aliases] test = pytest -[tool:pytest] -addopts = --verbose -r s - [versioneer] VCS = git style = pep440 @@ -12,12 +9,82 @@ versionfile_build = gwvet/_version.py tag_prefix = parentdir_prefix = +[metadata] +name = gwvet +author = Alex Urban, Duncan Macleod +author_email = alexander.urban@ligo.org +license = GPL-3.0-or-later +license_file = LICENSE +keywords = physics, astronomy, gravitational-waves, ligo +url = https://github.com/gwpy/vet +description = An extension to the python toolbox GWSumm, used by the LIGO Scientific Collaboration to review data-quality vetoes +long_description = file: README.rst +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Intended Audience :: Science/Research + License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) + Natural Language :: English + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Topic :: Scientific/Engineering + Topic :: Scientific/Engineering :: Astronomy + Topic :: Scientific/Engineering :: Physics + +[options] +zip_safe = False +packages = find: +python_requires = >=3.6 +setup_requires = + setuptools >=30.3.0 +install_requires = + astropy >=3.0.0 + decorator + dqsegdb + gwdetchar >=2.0.0 + gwpy >=2.0.0 + gwsumm >=2.0.0 + gwtrigfind + lscsoft-glue >=1.60.0 + MarkupPy + matplotlib >=3.1 + numpy >=1.16 + scipy +tests_require = + flake8 + pytest >=3.3.0 + pytest-cov >=2.4.0 + +[options.entry_points] +console_scripts = + gwvet = gwvet.__main__:main + gwvet-hug = gwvet.hug:main + gwvet-vdf = gwvet.vdf:main + +[options.extras_require] +doc = + sphinx + numpydoc + sphinx_rtd_theme + sphinxcontrib-epydoc + +[tool:pytest] +; print skip reasons +addopts = --verbose -r s + +; -- tools ------------------ + [flake8] exclude = - __pycache__, - .eggs/, - .git/, - build/, - docs/, - gwsumm/_version.py, - versioneer.py, + __pycache__, + .eggs/, + .git/, + build/, + docs/, + gwvet/_version.py, + versioneer.py, diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 72fc4e3..725f633 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (C) Duncan Macleod (2013) # @@ -17,33 +16,16 @@ # You should have received a copy of the GNU General Public License # along with GWpy VET. If not, see . -"""Setup the GWpy VET (`gwvet`) package +"""Setup the GW Veto Evaluation and Testing (`gwvet`) package """ -import sys -import glob -import os.path - -from setuptools import (setup, find_packages) - -# set basic metadata -PACKAGENAME = 'gwvet' -DISTNAME = 'gwvet' -AUTHOR = 'Duncan Macleod, Alex Urban' -AUTHOR_EMAIL = 'alexander.urban@ligo.org' -LICENSE = 'GPL-3.0-or-later' - -cmdclass = {} - -# -- versioning --------------------------------------------------------------- - import versioneer -__version__ = versioneer.get_version() -cmdclass.update(versioneer.get_cmdclass()) -# -- documentation ------------------------------------------------------------ +from setuptools import setup + +version = versioneer.get_version() +cmdclass = versioneer.get_cmdclass() -# import sphinx commands try: from sphinx.setup_command import BuildDoc except ImportError: @@ -51,80 +33,14 @@ else: cmdclass['build_sphinx'] = BuildDoc -# -- dependencies ------------------------------------------------------------- - -setup_requires = ['pytest-runner'] if { - 'pytest', 'test'}.intersection(sys.argv) else [] -install_requires = [ - 'astropy>=1.0', - 'decorator', - 'dqsegdb', - 'gwdetchar>=1.0.0', - 'gwpy>=1.0.0', - 'gwsumm>=1.0.1', - 'gwtrigfind', - 'lscsoft-glue>=1.60.0', - 'MarkupPy', - 'matplotlib>=1.5', - 'numpy>=1.7', - 'scipy', -] -tests_require = [ - 'pytest>=2.8,<3.7', - 'pytest-cov', - 'flake8', -] -extras_require = { - 'doc': ['sphinx', 'numpydoc', 'sphinx_rtd_theme', 'sphinxcontrib-epydoc'], -} - -# -- run setup ---------------------------------------------------------------- - -packagenames = find_packages() -scripts = glob.glob(os.path.join('bin', '*')) - -# read description -with open('README.rst', 'rb') as f: - longdesc = f.read().decode().strip() - +# run setup +# NOTE: all other metadata and options come from setup.cfg setup( - name=DISTNAME, - provides=[PACKAGENAME], - version=__version__, - description=("an extension to the python toolbox GWSumm, used by the LIGO " - "Scientific Collaboration to review data-quality vetoes"), - long_description=longdesc, - author=AUTHOR, - author_email=AUTHOR_EMAIL, - license=LICENSE, - url='https://github.com/gwpy/vet', + version=version, project_urls={ "Bug Tracker": "https://github.com/gwpy/vet/issues", "Discussion Forum": "https://gwdetchar.slack.com", "Source Code": "https://github.com/gwpy/vet", }, - packages=packagenames, - include_package_data=True, cmdclass=cmdclass, - scripts=scripts, - setup_requires=setup_requires, - install_requires=install_requires, - tests_require=tests_require, - extras_require=extras_require, - use_2to3=False, - classifiers=[ - 'Programming Language :: Python', - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Science/Research', - ('License :: OSI Approved :: ' - 'GNU General Public License v3 or later (GPLv3+)'), - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Scientific/Engineering :: Astronomy', - 'Topic :: Scientific/Engineering :: Physics', - ], )