WIP: Check full seed

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,12 @@ assignees: ''
 
 ---
 
+<!--
+Please fill in the following
+
+To get versions of python packages, import them and `print(package.__version__)`
+-->
+
 **Describe the bug**
 A clear and concise description of what the bug is.
 

.github/ISSUE_TEMPLATE/help-request.md

@@ -9,6 +9,8 @@ assignees: ''
 
 <!--
 Please fill out the form below so that we can help as best we can
+
+To get versions of python packages, import them and `print(package.__version__)`
 -->
 
 # What do you need help with?

eqcorrscan/core/lag_calc.py

@@ -190,7 +190,8 @@ def _concatenate_and_correlate(streams, template, cores):
 def xcorr_pick_family(family, stream, shift_len=0.2, min_cc=0.4,
                       horizontal_chans=['E', 'N', '1', '2'],
                       vertical_chans=['Z'], cores=1, interpolate=False,
-                      plot=False, plotdir=None, export_cc=False, cc_dir=None):
+                      plot=False, plotdir=None, export_cc=False, cc_dir=None,
+                      check_full_seed=False):
     """
     Compute cross-correlation picks for detections in a family.
 
@@ -234,13 +235,20 @@ def xcorr_pick_family(family, stream, shift_len=0.2, min_cc=0.4,
     :type cc_dir: str
     :param cc_dir:
         Path to saving folder, NumPy files will be output here.
+        :type check_full_seed: bool
+    :param check_full_seed:
+        If True, will check for duplicate traces against the full SEED id,
+        including Network, Station, Location and Channel. If False (default),
+        will check only against Station and Channel.
+
 
     :return: Dictionary of picked events keyed by detection id.
     """
     picked_dict = {}
     delta = family.template.st[0].stats.delta
     detect_streams_dict = _prepare_data(
-        family=family, detect_data=stream, shift_len=shift_len)
+        family=family, detect_data=stream, shift_len=shift_len,
+        check_full_seed=check_full_seed)
     detection_ids = list(detect_streams_dict.keys())
     detect_streams = [detect_streams_dict[detection_id]
                       for detection_id in detection_ids]
@@ -349,7 +357,7 @@ def xcorr_pick_family(family, stream, shift_len=0.2, min_cc=0.4,
     return picked_dict
 
 
-def _prepare_data(family, detect_data, shift_len):
+def _prepare_data(family, detect_data, shift_len, check_full_seed):
     """
     Prepare data for lag_calc - reduce memory here.
 
@@ -360,6 +368,11 @@ def _prepare_data(family, detect_data, shift_len):
     :param detect_data: Stream to extract detection streams from.
     :type shift_len: float
     :param shift_len: Shift length in seconds allowed for picking.
+    :type check_full_seed: bool
+    :param check_full_seed:
+        If True, will check for duplicate traces against the full SEED id,
+        including Network, Station, Location and Channel. If False (default),
+        will check only against Station and Channel.
 
     :returns: Dictionary of detect_streams keyed by detection id
               to be worked on
@@ -385,8 +398,11 @@ def _prepare_data(family, detect_data, shift_len):
                 detect_stream.remove(trace)
                 Logger.warning("Masked array found for {0}, not supported, "
                                "removing.".format(trace.id))
-        stachans = [(tr.stats.station, tr.stats.channel)
-                    for tr in detect_stream]
+        if check_full_seed:
+            stachans = [tr.id for tr in detect_stream]
+        else:
+            stachans = [(tr.stats.station, tr.stats.channel)
+                        for tr in detect_stream]
         c_stachans = Counter(stachans)
         for key in c_stachans.keys():
             if c_stachans[key] > 1:
@@ -405,7 +421,8 @@ def _prepare_data(family, detect_data, shift_len):
 def lag_calc(detections, detect_data, template_names, templates,
              shift_len=0.2, min_cc=0.4, horizontal_chans=['E', 'N', '1', '2'],
              vertical_chans=['Z'], cores=1, interpolate=False,
-             plot=False, plotdir=None, export_cc=False, cc_dir=None):
+             plot=False, plotdir=None, export_cc=False, cc_dir=None,
+             check_full_seed=False):
     """
     Cross-correlation derived picking of seismic events.
 
@@ -462,6 +479,11 @@ def lag_calc(detections, detect_data, template_names, templates,
     :type cc_dir: str
     :param cc_dir:
         Path to saving folder, NumPy files will be output here.
+        :type check_full_seed: bool
+    :param check_full_seed:
+        If True, will check for duplicate traces against the full SEED id,
+        including Network, Station, Location and Channel. If False (default),
+        will check only against Station and Channel.
 
     :returns:
         Catalog of events with picks.  No origin information is included.
@@ -546,7 +568,8 @@ def lag_calc(detections, detect_data, template_names, templates,
                 min_cc=min_cc, horizontal_chans=horizontal_chans,
                 vertical_chans=vertical_chans, interpolate=interpolate,
                 cores=cores, shift_len=shift_len, plot=plot, plotdir=plotdir,
-                export_cc=export_cc, cc_dir=cc_dir)
+                export_cc=export_cc, cc_dir=cc_dir,
+                check_full_seed=check_full_seed)
             initial_cat.update(template_dict)
     # Order the catalogue to match the input
     output_cat = Catalog()

eqcorrscan/core/match_filter/family.py

@@ -505,7 +505,7 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
                  cores=1, interpolate=False, plot=False, plotdir=None,
                  parallel=True, process_cores=None, ignore_length=False,
                  ignore_bad_data=False, export_cc=False, cc_dir=None,
-                 **kwargs):
+                 check_full_seed=False, **kwargs):
         """
         Compute picks based on cross-correlation alignment.
 
@@ -572,6 +572,12 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
         :type cc_dir: str
         :param cc_dir:
             Path to saving folder, NumPy files will be output here.
+        :type check_full_seed: bool
+        :param check_full_seed:
+            If True, will check for duplicate traces against the full SEED id,
+            including Network, Station, Location and Channel. If False (default),
+            will check only against Station and Channel.
+
 
         :returns:
             Catalog of events with picks.  No origin information is included.
@@ -605,7 +611,7 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
             min_cc=min_cc, horizontal_chans=horizontal_chans,
             vertical_chans=vertical_chans, cores=cores,
             interpolate=interpolate, plot=plot, plotdir=plotdir,
-            export_cc=export_cc, cc_dir=cc_dir)
+            export_cc=export_cc, cc_dir=cc_dir, check_full_seed=check_full_seed)
         catalog_out = Catalog([ev for ev in picked_dict.values()])
         for detection_id, event in picked_dict.items():
             for pick in event.picks:

eqcorrscan/core/match_filter/party.py

@@ -784,7 +784,7 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
                  cores=1, interpolate=False, plot=False, plotdir=None,
                  parallel=True, process_cores=None, ignore_length=False,
                  ignore_bad_data=False, export_cc=False, cc_dir=None,
-                 **kwargs):
+                 check_full_seed=False, **kwargs):
         """
         Compute picks based on cross-correlation alignment.
 
@@ -851,6 +851,13 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
             If False (default), errors will be raised if data are excessively
             gappy or are mostly zeros. If True then no error will be raised,
             but an empty trace will be returned (and not used in detection).
+        :type check_full_seed: bool
+        :param check_full_seed:
+            If True, will check for duplicate traces against the full SEED id,
+            including Network, Station, Location and Channel. If False (default),
+            will check only against Station and Channel. This behavior was
+            originally necessary to cope with some software (i.e. SEISAN) not
+            storing picks with full SEED info.
 
         :returns:
             Catalog of events with picks.  No origin information is included.
@@ -909,7 +916,8 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
                     export_cc=export_cc, cc_dir=cc_dir,
                     parallel=parallel, process_cores=process_cores,
                     ignore_bad_data=ignore_bad_data,
-                    ignore_length=ignore_length, **kwargs)
+                    ignore_length=ignore_length,
+                    check_full_seed=check_full_seed, **kwargs)
         return catalog
 
     @staticmethod

eqcorrscan/core/template_gen.py

@@ -56,7 +56,7 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order,
                  all_horiz=False, delayed=True, plot=False, plotdir=None,
                  return_event=False, min_snr=None, parallel=False,
                  num_cores=False, save_progress=False, skip_short_chans=False,
-                 **kwargs):
+                 check_full_seed=False, **kwargs):
     """
     Generate processed and cut waveforms for use as templates.
 
@@ -120,6 +120,13 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order,
         Whether to ignore channels that have insufficient length data or not.
         Useful when the quality of data is not known, e.g. when downloading
         old, possibly triggered data from a datacentre
+    :type check_full_seed: bool
+    :param check_full_seed:
+        If True, will check the trace header against the full SEED id,
+        including Network, Station, Location and Channel. If False (default),
+        will check only against Station and Channel. This behavior was
+        originally necessary to cope with some software (i.e. SEISAN) not
+        storing picks with full SEED info.
 
     :returns: List of :class:`obspy.core.stream.Stream` Templates
     :rtype: list
@@ -393,7 +400,7 @@ def template_gen(method, lowcut, highcut, samp_rate, filt_order,
             template = _template_gen(
                 event.picks, st, length, swin, prepick=prepick, plot=plot,
                 all_horiz=all_horiz, delayed=delayed, min_snr=min_snr,
-                plotdir=plotdir)
+                plotdir=plotdir, check_full_seed=check_full_seed)
             process_lengths.append(len(st[0].data) / samp_rate)
             temp_list.append(template)
             catalog_out += event
@@ -588,7 +595,7 @@ def _rms(array):
 
 def _template_gen(picks, st, length, swin='all', prepick=0.05,
                   all_horiz=False, delayed=True, plot=False, min_snr=None,
-                  plotdir=None):
+                  plotdir=None, check_full_seed=False):
     """
     Master function to generate a multiplexed template for a single event.
 
@@ -634,6 +641,13 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05,
     :param plotdir:
         The path to save plots to. If `plotdir=None` (default) then the figure
         will be shown on screen.
+    :type check_full_seed: bool
+    :param check_full_seed:
+        If True, will check the trace header against the full SEED id,
+        including Network, Station, Location and Channel. If False (default),
+        will check only against Station and Channel. This behavior was
+        originally necessary to cope with some software (i.e. SEISAN) not
+        storing picks with full SEED info.
 
     :returns: Newly cut template.
     :rtype: :class:`obspy.core.stream.Stream`
@@ -713,11 +727,29 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05,
     starttimes = []
     for _swin in swin:
         for tr in st:
-            starttime = {'station': tr.stats.station,
-                         'channel': tr.stats.channel, 'picks': []}
-            station_picks = [pick for pick in picks_copy
-                             if pick.waveform_id.station_code ==
-                             tr.stats.station]
+            if check_full_seed:
+                starttime = {'network': tr.stats.network,
+                             'station': tr.stats.station,
+                             'location': tr.stats.location,
+                             'channel': tr.stats.channel,
+                             'picks': []}
+                station_picks = [pick for pick in picks_copy
+                                 if pick.waveform_id.network_code ==
+                                 tr.stats.network and
+                                 pick.waveform_id.station_code ==
+                                 tr.stats.station and
+                                 pick.waveform_id.location_code ==
+                                 tr.stats.location]
+            else:
+                Logger.warning(
+                    'Not checking full SEED id compatibility between' +
+                    ' picks and waveforms. Checking full net.sta.loc.chan ' +
+                    'compatibility will be default behavior in future release')
+                starttime = {'station': tr.stats.station,
+                             'channel': tr.stats.channel, 'picks': []}
+                station_picks = [pick for pick in picks_copy
+                                 if pick.waveform_id.station_code ==
+                                 tr.stats.station]
             if _swin == 'P_all':
                 p_pick = [pick for pick in station_picks
                           if pick.phase_hint.upper()[0] == 'P']
@@ -772,8 +804,15 @@ def _template_gen(picks, st, length, swin='all', prepick=0.05,
     for _starttime in starttimes:
         Logger.info(f"Working on channel {_starttime['station']}."
                     f"{_starttime['channel']}")
-        tr = st.select(
-            station=_starttime['station'], channel=_starttime['channel'])[0]
+        if check_full_seed:
+            tr = st.select(
+                network=_starttime['network'],
+                station=_starttime['station'],
+                location=_starttime['location'],
+                channel=_starttime['channel'])[0]
+        else:
+            tr = st.select(
+                station=_starttime['station'], channel=_starttime['channel'])[0]
         Logger.info(f"Found Trace {tr}")
         used_tr = False
         for pick in _starttime['picks']:

eqcorrscan/doc/requirements.txt

@@ -1,5 +1,7 @@
 matplotlib
-sphinx_rtd_theme
 nbsphinx
 sphinx>=1.5.1
-sphinx_bootstrap_theme
+sphinx_bootstrap_theme
+# have to pin otherwise RTD fails
+# see https://github.com/readthedocs/readthedocs.org/issues/7492
+Pygments==2.6.1

eqcorrscan/doc/tutorials/quick_start.ipynb

@@ -397,7 +397,12 @@
     "\n",
     "We can also undertake cross-correlation phase-picking using our templates.  This is achieved\n",
     "using the `lag_calc` method on `Party` or `Family` objects.  You can also use the\n",
-    "`eqcorrscan.core.lag_calc` module directly on other catalogs."
+    "`eqcorrscan.core.lag_calc` module directly on other catalogs.\n",
+    "\n",
+    "We need to provide a merged stream to the lag-calc function. The stream we have obtained\n",
+    "from `tribe.detect` has overlaps in it. Using `stream.merge()` would result in gaps at those\n",
+    "overlaps, which we do not want. We use `stream.merge(method=1)` to take include the real data\n",
+    "in that gap."
    ]
   },
   {
@@ -415,7 +420,7 @@
     }
    ],
    "source": [
-    "st = st.merge()\n",
+    "st = st.merge(method=1)\n",
     "repicked_catalog = party.lag_calc(st, pre_processed=False, shift_len=0.5, min_cc=0.4)"
    ]
   },
@@ -516,17 +521,8 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.7.3"
-  },
-  "pycharm": {
-   "stem_cell": {
-    "cell_type": "raw",
-    "metadata": {
-     "collapsed": false
-    },
-    "source": []
-   }
   }
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

eqcorrscan/utils/catalog_to_dd.py

@@ -329,7 +329,8 @@ def _compute_dt(sparse_catalog, master, min_link, event_id_mapper):
         min_link=min_link) for event in sparse_catalog]
 
 
-def _make_event_pair(sparse_event, master, event_id_mapper, min_link):
+def _make_event_pair(sparse_event, master, event_id_mapper, min_link,
+                     check_full_seed=False):
     """
     Make an event pair for a given event and master event.
     """
@@ -339,9 +340,14 @@ def _make_event_pair(sparse_event, master, event_id_mapper, min_link):
     for master_pick in master.picks:
         if master_pick.phase  and master_pick.phase not in "PS":  # pragma: no cover
             continue
-        matched_picks = [p for p in sparse_event.picks
-                         if p.station == master_pick.station
-                         and p.phase == master_pick.phase]
+        if check_full_seed:
+            matched_picks = [p for p in sparse_event.picks
+                             if p.seed_id == master_pick.seed_id
+                             and p.phase == master_pick.phase]
+        else:
+            matched_picks = [p for p in sparse_event.picks
+                             if p.station == master_pick.station
+                             and p.phase == master_pick.phase]
         for matched_pick in matched_picks:
             differential_times.obs.append(
                 _DTObs(station=master_pick.station,