diff --git a/client/platform/desktop/backend/serializers/viame.ts b/client/platform/desktop/backend/serializers/viame.ts index 92cb48f51..94cb5e364 100644 --- a/client/platform/desktop/backend/serializers/viame.ts +++ b/client/platform/desktop/backend/serializers/viame.ts @@ -55,13 +55,28 @@ function getCaptureGroups(regexp: RegExp, str: string) { } function _rowInfo(row: string[]) { + let fps; if (row[0].match(CommentRegex) !== null) { - throw new Error('comment row'); + // we have a comment, check for FPS + let hasComment = false; + if (row.length > 1) { + if (row[1].startsWith('Fps:')) { + const fpsSplit = row[1].split(':'); + if (fpsSplit.length > 1) { + [, fps] = fpsSplit; + hasComment = true; + } + } + } + if (!hasComment) { + throw new Error('comment row'); + } } if (row.length < 9) { throw new Error('malformed row: too few columns'); } return { + fps, id: parseInt(row[0], 10), filename: row[1], frame: parseInt(row[2], 10), @@ -372,6 +387,13 @@ async function parse(input: Readable, imageMap?: Map): Promise<[ const { rowInfo, feature, trackAttributes, confidencePairs, } = _parseFeature(record); + if (rowInfo.fps) { + const parsedFps = parseInt(rowInfo.fps, 10); + if (!Number.isNaN(parsedFps)) { + fps = parsedFps; + } + throw new Error('comment row with FPS'); + } if (imageMap !== undefined) { const [imageName] = splitExt(rowInfo.filename); const expectedFrameNumber = imageMap.get(imageName); diff --git a/samples/scripts/setAnnotationFPS.py b/samples/scripts/setAnnotationFPS.py new file mode 100644 index 000000000..b0df96117 --- /dev/null +++ b/samples/scripts/setAnnotationFPS.py @@ -0,0 +1,54 @@ +import json +import os + +import click +import girder_client + +apiURL = "localhost" +rootFolder = "64c405c01cb3956240d67709" # Sample folder girder Id + + +# Login to the girder client, interactive means it will prompt for username and password +def login(): + gc = girder_client.GirderClient(apiURL, port=8010, apiRoot="girder/api/v1") + gc.authenticate(interactive=True) + return gc + + +def getFolderList(gc: girder_client.GirderClient, folderId, parentType="folder"): + folders = list(gc.listFolder(folderId, parentFolderType=parentType)) + return folders + +def process_folder(gc: girder_client.GirderClient, folderId, fps): + folders = getFolderList(gc,folderId) + processed = [] + for folder in folders: + if folder.get('meta', {}).get('annotate', False): # is a DIVE Dataset + old_annotation_fps = folder.get('meta', {},).get('fps', None) + video_fps = folder.get('meta', {},).get('originalFps', None) + gc.addMetadataToFolder(str(folder['_id']), { + "fps": int(fps) + }) + processed.append({ + 'name': folder.get('name', 'unknown'), + 'oldAnnotationFPS': old_annotation_fps, + 'newAnnotationFPS': int(fps), + 'videoFPS': video_fps, + }) + else: + processed = processed + process_folder(gc, str(folder['_id']), fps) + + return processed +@click.command(name="LoadData", help="Load in ") +@click.argument( + "fps" +) # An numerical FPS value to annotate bas +def load_data(fps): + gc = login() + # Search the root folder for a list of folders + processed = process_folder(gc, rootFolder, fps) + with open('processed.json', 'w', encoding='utf8') as outfile: + json.dump(processed, outfile, ensure_ascii=False, indent=True) + +if __name__ == "__main__": + load_data() diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 044bbba77..6dae2eb0b 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -326,10 +326,13 @@ def _get_data_by_type( # Parse the file as the now known type if as_type == crud.FileType.VIAME_CSV: - converted, attributes, warnings = viame.load_csv_as_tracks_and_attributes( + converted, attributes, warnings, fps = viame.load_csv_as_tracks_and_attributes( file_string.splitlines(), image_map ) - return {'annotations': converted, 'meta': None, 'attributes': attributes, 'type': as_type}, warnings + meta = None + if fps is not None: + meta = { "fps" : fps } + return {'annotations': converted, 'meta': meta, 'attributes': attributes, 'type': as_type}, warnings if as_type == crud.FileType.MEVA_KPF: converted, attributes = kpf.convert(kpf.load(file_string)) return {'annotations': converted, 'meta': None, 'attributes': attributes, 'type': as_type}, warnings @@ -349,7 +352,7 @@ def _get_data_by_type( return None, None -def process_items( +def process_items( folder: types.GirderModel, user: types.GirderUserModel, additive=False, diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index e1e66d92c..5a111e695 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -217,6 +217,7 @@ class MetadataMutable(BaseModel): confidenceFilters: Optional[Dict[str, float]] attributes: Optional[Dict[str, Attribute]] attributeTrackFilters: Optional[Dict[str, AttributeTrackFilter]] + fps: Optional[float] @staticmethod def is_dive_configuration(value: dict): diff --git a/server/dive_utils/serializers/viame.py b/server/dive_utils/serializers/viame.py index d4f5a231a..d1ae78ec2 100644 --- a/server/dive_utils/serializers/viame.py +++ b/server/dive_utils/serializers/viame.py @@ -280,7 +280,7 @@ def custom_sort(row): def load_csv_as_tracks_and_attributes( rows: List[str], imageMap: Optional[Dict[str, int]] = None, -) -> Tuple[types.DIVEAnnotationSchema, dict, List[str]]: +) -> Tuple[types.DIVEAnnotationSchema, dict, List[str], Optional[str]]: """ Convert VIAME CSV to json tracks @@ -296,9 +296,15 @@ def load_csv_as_tracks_and_attributes( foundImages: List[Dict[str, Any]] = [] # {image:str, frame: int, csvFrame: int} sortedlist = sorted(reader, key=custom_sort) warnings: List[str] = [] + fps = None for row in sortedlist: if len(row) == 0 or row[0].startswith('#'): # This is not a data row + if (len(row) > 0 and row[0] == '# metadata'): + if (row[1].startswith('Fps: ')): + fps_splits = row[1].split(':') + if len(fps_splits) > 1: + fps = fps_splits[1] continue ( feature, @@ -358,8 +364,6 @@ def load_csv_as_tracks_and_attributes( maxFrame = float('-inf') frameMapper = {} filteredImages = [item for item in foundImages if item['frame'] != -1] - print('IMAGEMAP') - print(imageMap) for index, item in enumerate(filteredImages): if item['frame'] == -1: continue @@ -456,7 +460,7 @@ def load_csv_as_tracks_and_attributes( 'groups': {}, 'version': constants.AnnotationsCurrentVersion, } - return annotations, metadata_attributes, warnings + return annotations, metadata_attributes, warnings, fps def export_tracks_as_csv( diff --git a/server/tests/test_attributes_processor.py b/server/tests/test_attributes_processor.py index 8a79a3096..d5dfae4b0 100644 --- a/server/tests/test_attributes_processor.py +++ b/server/tests/test_attributes_processor.py @@ -26,7 +26,7 @@ def test_read_viame_attributes( rows.append(line) text = text + line print() - converted, attributes, warnings = load_csv_as_tracks_and_attributes(text.split('\n')) + converted, attributes, warnings, fps = load_csv_as_tracks_and_attributes(text.split('\n')) assert json.dumps(converted['tracks'], sort_keys=True) == json.dumps( expected_tracks, sort_keys=True ) diff --git a/server/tests/test_deserialize_viame_csv.py b/server/tests/test_deserialize_viame_csv.py index a955f4846..4fd1d16fa 100644 --- a/server/tests/test_deserialize_viame_csv.py +++ b/server/tests/test_deserialize_viame_csv.py @@ -15,7 +15,7 @@ def test_read_viame_csv( expected_tracks: Dict[str, dict], expected_attributes: Dict[str, dict], ): - (converted, attributes, warnings) = viame.load_csv_as_tracks_and_attributes(input) + (converted, attributes, warnings, fps) = viame.load_csv_as_tracks_and_attributes(input) assert json.dumps(converted['tracks'], sort_keys=True) == json.dumps( expected_tracks, sort_keys=True ) diff --git a/server/tests/test_serialize_viame_csv.py b/server/tests/test_serialize_viame_csv.py index e70c8f2d2..aa065b3d6 100644 --- a/server/tests/test_serialize_viame_csv.py +++ b/server/tests/test_serialize_viame_csv.py @@ -354,9 +354,9 @@ def test_image_filenames(): image_map = {'1': 0, '2': 1, '3': 2} for test in image_filename_tests: if not test['warning']: - converted, _, warnings = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map) + converted, _, warnings, fps = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map) assert len(converted['tracks'].values()) > 0 else: - converted, _, warnings = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map) + converted, _, warnings, fps = viame.load_csv_as_tracks_and_attributes(test['csv'], image_map) assert len(warnings) > 0