Rework metrics

.gitignore

@@ -139,3 +139,6 @@ docs/PROJECTS.md
 
 # vim
 *~
+
+# sphinx created csvs
+docs/source/*.csv

.travis.yml

@@ -1,6 +1,5 @@
 language: python
 python:
-  - "3.6"
   - "3.7"
   - "3.8"
   - "3.9"

ChildProject/annotations.py

@@ -13,7 +13,7 @@
 from .projects import ChildProject
 from .converters import *
 from .tables import IndexTable, IndexColumn, assert_dataframe, assert_columns_presence
-from .utils import Segment, intersect_ranges, path_is_parent
+from .utils import Segment, intersect_ranges, path_is_parent, TimeInterval
 
 
 class AnnotationManager:
@@ -1191,7 +1191,7 @@ def get_within_ranges(
         return pd.concat(stack) if len(stack) else pd.DataFrame()
 
     def get_within_time_range(
-        self, annotations: pd.DataFrame, start_time: str, end_time: str, errors="raise"
+        self, annotations: pd.DataFrame, interval : TimeInterval, errors="raise"
     ):
         """Clip all input annotations within a given HH:MM clock-time range.
         Those that do not intersect the input time range at all are filtered out.
@@ -1222,24 +1222,12 @@ def get_within_time_range(
         def get_ms_since_midight(dt):
             return (dt - dt.replace(hour=0, minute=0, second=0)).total_seconds() * 1000
 
-        try:
-            start_dt = datetime.datetime.strptime(start_time, "%H:%M")
-        except:
-            raise ValueError(
-                f"invalid value for start_time ('{start_time}'); should have HH:MM format instead"
-            )
-
-        try:
-            end_dt = datetime.datetime.strptime(end_time, "%H:%M")
-        except:
-            raise ValueError(
-                f"invalid value for end_time ('{end_time}'); should have HH:MM format instead"
-            )
-
-        assert end_dt > start_dt, "end_time must follow start_time"
+        #assert end_dt > start_dt, "end_time must follow start_time"
+        # no reason to keep this condition, 23:00 to 03:00 is completely acceptable
 
-        start_ts = get_ms_since_midight(start_dt)
-        end_ts = get_ms_since_midight(end_dt)
+        if not isinstance(interval, TimeInterval): raise ValueError("interval must be a TimeInterval object")
+        start_ts = get_ms_since_midight(interval.start)
+        end_ts = get_ms_since_midight(interval.stop)
 
         annotations = annotations.merge(
             self.project.recordings[["recording_filename", "start_time"]], how="left"
@@ -1267,8 +1255,18 @@ def get_ms_since_midight(dt):
 
         matches = []
         for annotation in annotations.to_dict(orient="records"):
+            #onsets = np.arange(start_ts, annotation["range_offset_ts"], 86400 * 1000)
+            #offsets = onsets + (end_ts - start_ts)
+
             onsets = np.arange(start_ts, annotation["range_offset_ts"], 86400 * 1000)
-            offsets = onsets + (end_ts - start_ts)
+            offsets = np.arange(end_ts, annotation["range_offset_ts"], 86400 * 1000)
+            #treat edge cases when the offset is after the end of annotation, onset before start etc
+            if len(onsets) > 0 and onsets[0] < annotation["range_onset_ts"] :
+                if len(offsets) > 0 and offsets[0] < annotation["range_onset_ts"]: onsets = onsets[1:]
+                else : onsets[0] = annotation["range_onset_ts"]
+            if len(offsets) > 0 and offsets[0] < annotation["range_onset_ts"] : offsets = offsets[1:]
+            if len(onsets) > 0 and len(offsets) > 0 and onsets[0] > offsets[0] : onsets = np.append(annotation["range_onset_ts"], onsets)
+            if (len(onsets) > 0 and len(offsets) > 0 and onsets[-1] > offsets[-1]) or len(onsets) > len(offsets) : offsets = np.append(offsets,annotation["range_offset_ts"])
 
             xs = (Segment(onset, offset) for onset, offset in zip(onsets, offsets))
             ys = iter(

ChildProject/cmdline.py

@@ -513,6 +513,7 @@ def main():
     register_pipeline("eaf-builder", EafBuilderPipeline)
     register_pipeline("anonymize", AnonymizationPipeline)
     register_pipeline("metrics", MetricsPipeline)
+    register_pipeline("metrics-specification", MetricsSpecificationPipeline)
 
     args = parser.parse_args()
     args.func(args)

ChildProject/pipelines/__init__.py

@@ -2,5 +2,6 @@
 from .eafbuilder import EafBuilderPipeline
 from .zooniverse import ZooniversePipeline
 from .metrics import MetricsPipeline
+from .metrics import MetricsSpecificationPipeline
 from .processors import AudioProcessingPipeline
 from .anonymize import AnonymizationPipeline