From 72754bb426c82cf7bc9de297996336350d8fa24d Mon Sep 17 00:00:00 2001 From: Evan Goetz Date: Thu, 16 Mar 2023 15:44:47 -0700 Subject: [PATCH 1/3] Added a new "metastate" section type where one can combine states from different detectors This fixes a problem where the "Quiet" and "Noisy" states were undefined in time, so the whole duration was used. This meant there was no distinction between the states as plotted for the summary pages. A metastate would be defined as ``` [metastate-Quiet] key = Quiet name = SEI Quiet uses = H1-quiet,L1-quiet [state-H1-Quiet] name = SEI Quiet key = H1-quiet hours = 1-5,H1 [state-L1-Quiet] name = SEI Quiet key = L1-quiet hours = 2-6,L1 ``` It is not super robust in that it's really only supposed to be used for combining some interval for H1 and some interval for L1. I'm not sure it's really ready to be heavily used in many situations Addresses the comment in #358 --- gwsumm/config.py | 5 ++++- gwsumm/plot/builtin.py | 6 +++++- gwsumm/state/__init__.py | 4 ++-- gwsumm/state/core.py | 42 ++++++++++++++++++++++++++++++++++++++++ gwsumm/tabs/data.py | 18 ++++++++++++++--- 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/gwsumm/config.py b/gwsumm/config.py index e5195bde..3a904e31 100644 --- a/gwsumm/config.py +++ b/gwsumm/config.py @@ -305,7 +305,7 @@ def load_states(self, section='states'): """Read and format a list of `SummaryState` definitions from the given :class:`~configparser.ConfigParser` """ - from .state import (register_state, SummaryState, + from .state import (register_state, SummaryState, SummaryMetaState, ALLSTATE, generate_all_state, get_state) # parse the [states] section into individual state definitions try: @@ -327,6 +327,9 @@ def load_states(self, section='states'): if re.match(r'state[-\s]', section): states.append(register_state( SummaryState.from_ini(self, section))) + elif re.match(r'metastate[-\s]', section): + states.append(register_state( + SummaryMetaState.from_ini(self, section))) # register All state start = self.getint(section, 'gps-start-time') diff --git a/gwsumm/plot/builtin.py b/gwsumm/plot/builtin.py index 54fa5020..92800e63 100644 --- a/gwsumm/plot/builtin.py +++ b/gwsumm/plot/builtin.py @@ -41,7 +41,7 @@ from ..data import (get_timeseries, get_spectrogram, get_coherence_spectrogram, get_range_spectrogram, get_spectrum, get_coherence_spectrum, get_range_spectrum) -from ..state import ALLSTATE +from ..state import ALLSTATE, SummaryMetaState from .registry import (get_plot, register_plot) from .mixins import DataLabelSvgMixin @@ -508,6 +508,10 @@ def _draw(self): else: valid = SegmentList([self.span]) + if isinstance(valid, SummaryMetaState): + valid = globalv.STATES[ + f'{channel.ifo}-{valid.uses[0][3:]}'.lower()] + if self.type == 'coherence-spectrum': data = get_coherence_spectrum( [str(channel), str(channel2)], valid, query=False) diff --git a/gwsumm/state/__init__.py b/gwsumm/state/__init__.py index 860f3dfe..250e0b53 100644 --- a/gwsumm/state/__init__.py +++ b/gwsumm/state/__init__.py @@ -55,9 +55,9 @@ """ -from .core import SummaryState +from .core import (SummaryState, SummaryMetaState) from .registry import (get_state, get_states, register_state) from .all import (ALLSTATE, generate_all_state) __all__ = ['ALLSTATE', 'SummaryState', 'get_state', 'get_states', - 'register_state', 'generate_all_state'] + 'register_state', 'generate_all_state', 'SummaryMetaState'] diff --git a/gwsumm/state/core.py b/gwsumm/state/core.py index f5c64310..7c64798d 100644 --- a/gwsumm/state/core.py +++ b/gwsumm/state/core.py @@ -349,3 +349,45 @@ def copy(self): def __str__(self): return self.name + + +class SummaryMetaState(SummaryState): + """A meta state where different states may be used""" + + def __init__(self, name, known=SegmentList(), active=SegmentList(), + description=None, definition=None, hours=None, key=None, + filename=None, url=None, uses=[]): + + super(SummaryMetaState, self).__init__( + name=name, known=known, active=active, + description=description, definition=definition, hours=hours, + key=key, filename=filename, url=url) + + self.uses = uses + + @classmethod + def from_ini(cls, config, section): + config = GWSummConfigParser.from_configparser(config) + # get parameters + params = dict(config.nditems(section)) + # parse name + name = params.pop('name', section) + if re.match(r'metastate[-\s]', name): + name = section[10:] + # list states this uses + uses = params.pop('uses', section).split(',') + + # generate metastate + return cls(name=name, uses=uses, **params) + + def fetch(self, config=GWSummConfigParser(), + segmentcache=None, segdb_error='raise', + datacache=None, datafind_error='raise', nproc=1, nds=None, + **kwargs): + + for idx, state in enumerate(self.uses): + globalv.STATES[state.lower()].fetch( + config=config, segmentcache=segmentcache, + segdb_error=segdb_error, datacache=datacache, + datafind_error=datafind_error, nproc=nproc, nds=nds, + **kwargs) diff --git a/gwsumm/tabs/data.py b/gwsumm/tabs/data.py index aec3e7d8..3cf81a78 100644 --- a/gwsumm/tabs/data.py +++ b/gwsumm/tabs/data.py @@ -60,7 +60,8 @@ from ..data.utils import get_fftparams from ..plot import get_plot from ..segments import get_segments -from ..state import (generate_all_state, ALLSTATE, get_state) +from ..state import (generate_all_state, ALLSTATE, get_state, + SummaryMetaState) from ..triggers import get_triggers from ..utils import (re_flagdiv, vprint, safe_eval) @@ -357,11 +358,22 @@ def process(self, config=ConfigParser(), nproc=1, **stateargs): nproc=nproc, nds=stateargs.get('nds', None)) vprint("States finalised [%d total]\n" % len(self.states)) for state in self.states: - vprint(" {0.name}: {1} segments | {2} seconds".format( - state, len(state.active), abs(state.active))) + if isinstance(state, SummaryMetaState): + vprint( + f"Metastate {state.key} has {len(state.uses)} states") + else: + vprint(" {0.name}: {1} segments | {2} seconds".format( + state, len(state.active), abs(state.active))) if state is self.defaultstate: vprint(" [DEFAULT]") vprint('\n') + if isinstance(state, SummaryMetaState): + for idx, this_state in enumerate(state.uses): + vprint(f" {this_state}: ") + vprint(f"{len(globalv.STATES[this_state.lower()].active)} " + "segments | ") + vprint(f"{abs(globalv.STATES[this_state.lower()].active)} " + "seconds\n") # pre-process requests for 'all-data' plots all_data = any([(p.all_data & p.new) for p in self.plots]) From b10b53cf00f4abae7bb9b29b3986301fb50cc2d4 Mon Sep 17 00:00:00 2001 From: Evan Goetz Date: Fri, 17 Mar 2023 09:30:48 -0700 Subject: [PATCH 2/3] Add unit test --- gwsumm/tests/test_config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gwsumm/tests/test_config.py b/gwsumm/tests/test_config.py index ffee7563..4b6fedfb 100644 --- a/gwsumm/tests/test_config.py +++ b/gwsumm/tests/test_config.py @@ -186,6 +186,18 @@ def test_load_states(self): assert states['locked'].definition == 'X1:TEST-STATE:1' assert state.ALLSTATE in states + def test_load_state_metastate(self): + cp = self.new() + cp.set_date_options(0, 100) + cp.add_section('metastate-test') + cp.set('metastate-test', 'uses', 'locked') + cp.set('metastate-test', 'name', 'meta') + cp.load_states() + states = state.get_states() + assert len(states) == 3 + assert 'meta' in states + assert states['meta'].uses == ['locked'] + def test_load_plugins(self, cnfg): # check that empty section doesn't cause havoc cp = self.PARSER() From 77d68edda68ab38ef33c0979de1d7f0c57e74ea8 Mon Sep 17 00:00:00 2001 From: Evan Goetz Date: Fri, 17 Mar 2023 10:27:03 -0700 Subject: [PATCH 3/3] Improve state handling when using metastates and add better documentation and comments --- gwsumm/config.py | 19 ++++++++++---- gwsumm/plot/builtin.py | 17 +++++++++++-- gwsumm/state/core.py | 56 +++++++++++++++++++++++++++++++++++++++--- gwsumm/tabs/data.py | 2 ++ 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/gwsumm/config.py b/gwsumm/config.py index 3a904e31..7826f57e 100644 --- a/gwsumm/config.py +++ b/gwsumm/config.py @@ -307,21 +307,30 @@ def load_states(self, section='states'): """ from .state import (register_state, SummaryState, SummaryMetaState, ALLSTATE, generate_all_state, get_state) - # parse the [states] section into individual state definitions + # Parse the [states] section into individual state definitions. + # Each state definition is amended to the GWSummConfigParser as a new + # section with name and definition key-value pairs. try: states = dict(self.nditems(section)) except configparser.NoSectionError: self.add_section(section) states = {} for state in states: - if not (self.has_section('state-%s' % state) or - self.has_section('state %s' % state)): - section = 'state-%s' % state + if not (self.has_section(f'state-{state}') or + self.has_section(f'state {state}')): + section = f'state-{state}' self.add_section(section) self.set(section, 'name', state) self.set(section, 'definition', states[state]) - # parse each state section into a new state + # Parse each state section into a new state. + # Here we reset the states variable to an empty list because the + # previous code block added all of the states section into their own + # sections [state-]. We register those states and metastates, + # appending them also to the states list. Metastates are states that + # use another state definition, where they look for the key name. If + # key is not defined, then name is used instead. The key or name is + # expected to have an 'H1-' or 'L1-' prefix. states = [] for section in self.sections(): if re.match(r'state[-\s]', section): diff --git a/gwsumm/plot/builtin.py b/gwsumm/plot/builtin.py index 92800e63..138826aa 100644 --- a/gwsumm/plot/builtin.py +++ b/gwsumm/plot/builtin.py @@ -20,6 +20,7 @@ """ import os.path +import re import warnings from itertools import cycle @@ -498,19 +499,31 @@ def _draw(self): else: iterator = list(zip(self.channels, plotargs)) + # loop over the channels for chantuple in iterator: channel = chantuple[0] channel2 = chantuple[1] pargs = chantuple[-1] + # get the state or segment if self.state and not self.all_data: valid = self.state else: valid = SegmentList([self.span]) + # If the state is a metastate, then get the corresponding IFO- + # specific state from the global variables list if isinstance(valid, SummaryMetaState): - valid = globalv.STATES[ - f'{channel.ifo}-{valid.uses[0][3:]}'.lower()] + reg = re.compile(channel.ifo) + matching = list(filter(reg.match, valid.uses)) + assert len(matching) == 1, ( + f"Failed to find a unique state for {valid.name} " + f"metastate. Found {len(matching)} matching states in " + f"{valid.uses} for {channel.ifo}") + try: + valid = globalv.STATES[matching[0].lower()] + except KeyError: + raise if self.type == 'coherence-spectrum': data = get_coherence_spectrum( diff --git a/gwsumm/state/core.py b/gwsumm/state/core.py index 7c64798d..2ce40a28 100644 --- a/gwsumm/state/core.py +++ b/gwsumm/state/core.py @@ -352,11 +352,43 @@ def __str__(self): class SummaryMetaState(SummaryState): - """A meta state where different states may be used""" + """A meta state where different states may be used when processing a + `~gwsumm.tabs.DataTab`. - def __init__(self, name, known=SegmentList(), active=SegmentList(), + An example use ase is when one wants to plot two different state times on + the same plot. This currently has limitations as it expects the states to + be from different detectors. So when using this metastate, each value in + "uses" needs to be prefixed by "" + + Parameters + ---------- + name : `str` + name for this state + uses : `list` + list of strings for which states to use. Ex.: ['H1-quiet', 'L1-quiet'] + known : `~gwpy.segments.SegmentList`, optional + list of known segments + active : `~gwpy.segments.SegmentList`, optional + list of active segments + description : `str`, optional + text describing what this state means + definition : `str`, optional + logical combination of flags that define known and active segments + for this state (see :attr:`documentation ` + for details) + hours : `str`, optional + a string of the form "-," + key : `str`, optional + registry key for this state, defaults to :attr:`~SummaryState.name` + filename : `str`, optional + path to filename with segments + url : `str`, optional + URL to read the segments + """ + + def __init__(self, name, uses, known=SegmentList(), active=SegmentList(), description=None, definition=None, hours=None, key=None, - filename=None, url=None, uses=[]): + filename=None, url=None): super(SummaryMetaState, self).__init__( name=name, known=known, active=active, @@ -367,6 +399,21 @@ def __init__(self, name, known=SegmentList(), active=SegmentList(), @classmethod def from_ini(cls, config, section): + """Create a new `SummaryMetaState` from a section in a `ConfigParser`. + + Parameters + ---------- + config : :class:`~gwsumm.config.GWConfigParser` + customised configuration parser containing given section + section : `str` + name of section to parse + + Returns + ------- + `SummaryMetaState` + a new state, with attributes set from the options in the + configuration + """ config = GWSummConfigParser.from_configparser(config) # get parameters params = dict(config.nditems(section)) @@ -384,6 +431,9 @@ def fetch(self, config=GWSummConfigParser(), segmentcache=None, segdb_error='raise', datacache=None, datafind_error='raise', nproc=1, nds=None, **kwargs): + """Finalise this state by fetching the states this metastate uses, + either from global memory, or from the segment database + """ for idx, state in enumerate(self.uses): globalv.STATES[state.lower()].fetch( diff --git a/gwsumm/tabs/data.py b/gwsumm/tabs/data.py index 3cf81a78..c84f3737 100644 --- a/gwsumm/tabs/data.py +++ b/gwsumm/tabs/data.py @@ -357,6 +357,8 @@ def process(self, config=ConfigParser(), nproc=1, **stateargs): datafind_error=stateargs.get('datafind_error', 'raise'), nproc=nproc, nds=stateargs.get('nds', None)) vprint("States finalised [%d total]\n" % len(self.states)) + + # loop over states for this tab and print out information for state in self.states: if isinstance(state, SummaryMetaState): vprint(