reading FPS from CSV Metadata

client/platform/desktop/backend/serializers/viame.ts

@@ -55,13 +55,28 @@ function getCaptureGroups(regexp: RegExp, str: string) {
 }
 
 function _rowInfo(row: string[]) {
+  let fps;
   if (row[0].match(CommentRegex) !== null) {
-    throw new Error('comment row');
+    // we have a comment, check for FPS
+    let hasComment = false;
+    if (row.length > 1) {
+      if (row[1].startsWith('Fps:')) {
+        const fpsSplit = row[1].split(':');
+        if (fpsSplit.length > 1) {
+          [, fps] = fpsSplit;
+          hasComment = true;
+        }
+      }
+    }
+    if (!hasComment) {
+      throw new Error('comment row');
+    }
   }
   if (row.length < 9) {
     throw new Error('malformed row: too few columns');
   }
   return {
+    fps,
     id: parseInt(row[0], 10),
     filename: row[1],
     frame: parseInt(row[2], 10),
@@ -372,6 +387,13 @@ async function parse(input: Readable, imageMap?: Map<string, number>): Promise<[
           const {
             rowInfo, feature, trackAttributes, confidencePairs,
           } = _parseFeature(record);
+          if (rowInfo.fps) {
+            const parsedFps = parseInt(rowInfo.fps, 10);
+            if (!Number.isNaN(parsedFps)) {
+              fps = parsedFps;
+            }
+            throw new Error('comment row with FPS');
+          }
           if (imageMap !== undefined) {
             const [imageName] = splitExt(rowInfo.filename);
             const expectedFrameNumber = imageMap.get(imageName);

samples/scripts/setAnnotationFPS.py

@@ -0,0 +1,54 @@
+import json
+import os
+
+import click
+import girder_client
+
+apiURL = "localhost"
+rootFolder = "64c405c01cb3956240d67709"  # Sample folder girder Id
+
+
+# Login to the girder client, interactive means it will prompt for username and password
+def login():
+    gc = girder_client.GirderClient(apiURL, port=8010, apiRoot="girder/api/v1")
+    gc.authenticate(interactive=True)
+    return gc
+
+
+def getFolderList(gc: girder_client.GirderClient, folderId, parentType="folder"):
+    folders = list(gc.listFolder(folderId, parentFolderType=parentType))
+    return folders
+
+def process_folder(gc: girder_client.GirderClient, folderId, fps):
+    folders = getFolderList(gc,folderId)
+    processed = []
+    for folder in folders:
+        if folder.get('meta', {}).get('annotate', False):  # is a DIVE Dataset
+            old_annotation_fps = folder.get('meta', {},).get('fps', None)
+            video_fps = folder.get('meta', {},).get('originalFps', None)
+            gc.addMetadataToFolder(str(folder['_id']), {
+                "fps": int(fps)
+            })
+            processed.append({
+                'name': folder.get('name', 'unknown'),
+                'oldAnnotationFPS': old_annotation_fps,
+                'newAnnotationFPS': int(fps),
+                'videoFPS': video_fps,
+            })
+        else:
+            processed = processed + process_folder(gc, str(folder['_id']), fps)
+
+    return processed
+@click.command(name="LoadData", help="Load in ")
+@click.argument(
+    "fps"
+)  # An numerical FPS value to annotate bas
+def load_data(fps):
+    gc = login()
+    # Search the root folder for a list of folders
+    processed = process_folder(gc, rootFolder, fps)
+    with open('processed.json', 'w', encoding='utf8') as outfile:
+        json.dump(processed, outfile, ensure_ascii=False, indent=True)
+
+if __name__ == "__main__":
+    load_data()

server/dive_server/crud_rpc.py

@@ -326,10 +326,13 @@ def _get_data_by_type(
 
     # Parse the file as the now known type
     if as_type == crud.FileType.VIAME_CSV:
-        converted, attributes, warnings = viame.load_csv_as_tracks_and_attributes(
+        converted, attributes, warnings, fps = viame.load_csv_as_tracks_and_attributes(
             file_string.splitlines(), image_map
         )
-        return {'annotations': converted, 'meta': None, 'attributes': attributes, 'type': as_type}, warnings
+        meta = None
+        if fps is not None:
+            meta = { "fps" : fps }
+        return {'annotations': converted, 'meta': meta, 'attributes': attributes, 'type': as_type}, warnings
     if as_type == crud.FileType.MEVA_KPF:
         converted, attributes = kpf.convert(kpf.load(file_string))
         return {'annotations': converted, 'meta': None, 'attributes': attributes, 'type': as_type}, warnings
@@ -349,7 +352,7 @@ def _get_data_by_type(
     return None, None
 
 
-def process_items(
+def process_items( 
     folder: types.GirderModel,
     user: types.GirderUserModel,
     additive=False,

server/dive_utils/models.py

@@ -217,6 +217,7 @@ class MetadataMutable(BaseModel):
     confidenceFilters: Optional[Dict[str, float]]
     attributes: Optional[Dict[str, Attribute]]
     attributeTrackFilters: Optional[Dict[str, AttributeTrackFilter]]
+    fps: Optional[float]
 
     @staticmethod
     def is_dive_configuration(value: dict):

server/dive_utils/serializers/viame.py

@@ -280,7 +280,7 @@ def custom_sort(row):
 
 def load_csv_as_tracks_and_attributes(
     rows: List[str], imageMap: Optional[Dict[str, int]] = None,
-) -> Tuple[types.DIVEAnnotationSchema, dict, List[str]]:
+) -> Tuple[types.DIVEAnnotationSchema, dict, List[str], Optional[str]]:
     """
     Convert VIAME CSV to json tracks
 
@@ -296,9 +296,15 @@ def load_csv_as_tracks_and_attributes(
     foundImages: List[Dict[str, Any]] = []  # {image:str, frame: int, csvFrame: int}
     sortedlist = sorted(reader, key=custom_sort)
     warnings: List[str] = []
+    fps = None
     for row in sortedlist:
         if len(row) == 0 or row[0].startswith('#'):
             # This is not a data row
+            if (len(row) > 0 and row[0] == '# metadata'):
+                if (row[1].startswith('Fps: ')):
+                    fps_splits = row[1].split(':')
+                    if len(fps_splits) > 1:
+                        fps = fps_splits[1]
             continue
         (
             feature,
@@ -358,8 +364,6 @@ def load_csv_as_tracks_and_attributes(
         maxFrame = float('-inf')
         frameMapper = {}
         filteredImages = [item for item in foundImages if item['frame'] != -1]
-        print('IMAGEMAP')
-        print(imageMap)
         for index, item in enumerate(filteredImages):
             if item['frame'] == -1:
                 continue
@@ -456,7 +460,7 @@ def load_csv_as_tracks_and_attributes(
         'groups': {},
         'version': constants.AnnotationsCurrentVersion,
     }
-    return annotations, metadata_attributes, warnings
+    return annotations, metadata_attributes, warnings, fps
 
 
 def export_tracks_as_csv(

server/tests/test_attributes_processor.py

@@ -26,7 +26,7 @@ def test_read_viame_attributes(
         rows.append(line)
         text = text + line
         print()
-    converted, attributes, warnings = load_csv_as_tracks_and_attributes(text.split('\n'))
+    converted, attributes, warnings, fps = load_csv_as_tracks_and_attributes(text.split('\n'))
     assert json.dumps(converted['tracks'], sort_keys=True) == json.dumps(
         expected_tracks, sort_keys=True
     )

server/tests/test_deserialize_viame_csv.py

@@ -15,7 +15,7 @@ def test_read_viame_csv(
     expected_tracks: Dict[str, dict],
     expected_attributes: Dict[str, dict],
 ):
-    (converted, attributes, warnings) = viame.load_csv_as_tracks_and_attributes(input)
+    (converted, attributes, warnings, fps) = viame.load_csv_as_tracks_and_attributes(input)
     assert json.dumps(converted['tracks'], sort_keys=True) == json.dumps(
         expected_tracks, sort_keys=True
     )

server/tests/test_serialize_viame_csv.py

@@ -354,9 +354,9 @@ def test_image_filenames():
     image_map = {'1': 0, '2': 1, '3': 2}
     for test in image_filename_tests:
         if not test['warning']:
-            converted, _, warnings = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map)
+            converted, _, warnings, fps = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map)
             assert len(converted['tracks'].values()) > 0
         else:
-            converted, _, warnings = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map)
+            converted, _, warnings, fps = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map)
             assert len(warnings) > 0