From e3ebe90b7f4cb641300275fe4d2e17abcca0aa21 Mon Sep 17 00:00:00 2001
From: Daan Krol <daan.krol@intel.com>
Date: Wed, 23 Oct 2024 10:56:36 +0200
Subject: [PATCH 1/5] Allow asynchronous annotation upload

---
 .../base_annotation_client.py                 | 104 +++++++++++-------
 1 file changed, 66 insertions(+), 38 deletions(-)

diff --git a/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py b/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
index 9dcd623c..2c039442 100644
--- a/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
+++ b/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
@@ -39,6 +39,7 @@
 from geti_sdk.http_session import GetiSession
 from geti_sdk.rest_clients.dataset_client import DatasetClient
 from geti_sdk.rest_converters import AnnotationRESTConverter
+from geti_sdk.http_session import GetiRequestException
 
 AnnotationReaderType = TypeVar("AnnotationReaderType", bound=AnnotationReader)
 MediaType = TypeVar("MediaType", Image, Video)
@@ -50,11 +51,11 @@ class BaseAnnotationClient:
     """
 
     def __init__(
-        self,
-        session: GetiSession,
-        workspace_id: str,
-        project: Project,
-        annotation_reader: Optional[AnnotationReaderType] = None,
+            self,
+            session: GetiSession,
+            workspace_id: str,
+            project: Project,
+            annotation_reader: Optional[AnnotationReaderType] = None,
     ):
         self.session = session
         self.workspace_id = workspace_id
@@ -70,7 +71,7 @@ def __init__(
         )
 
     def _get_all_media_by_type(
-        self, media_type: Type[MediaType]
+            self, media_type: Type[MediaType]
     ) -> MediaList[MediaType]:
         """
         Get a list holding all media entities of type `media_type` in the project.
@@ -89,7 +90,7 @@ def _get_all_media_by_type(
         return media_list
 
     def _get_all_media_in_dataset_by_type(
-        self, media_type: Type[MediaType], dataset: Dataset
+            self, media_type: Type[MediaType], dataset: Dataset
     ) -> MediaList[MediaType]:
         """
         Return a list of all media items of type `media_type` in the dataset `dataset`
@@ -163,13 +164,13 @@ def __get_label_mapping(self, project: Project) -> Dict[str, str]:
         # We include the project label names in the mapping, as we want to ensure that
         # we can match the labels from the source to the project labels.
         for source_label_name in source_label_names.union(
-            project_label_name_to_label.keys()
+                project_label_name_to_label.keys()
         ):
             if source_label_name in project_label_name_to_label:
                 label_id = project_label_name_to_label[source_label_name].id
             elif (
-                source_label_name.casefold() in project_label_name_to_label
-                and project_label_name_to_label[source_label_name.casefold()].is_empty
+                    source_label_name.casefold() in project_label_name_to_label
+                    and project_label_name_to_label[source_label_name.casefold()].is_empty
             ):
                 label_id = project_label_name_to_label[source_label_name.casefold()].id
             else:
@@ -202,9 +203,9 @@ def label_mapping(self) -> Dict[str, str]:
             )
 
     def _upload_annotation_for_2d_media_item(
-        self,
-        media_item: Union[Image, VideoFrame],
-        annotation_scene: Optional[AnnotationScene] = None,
+            self,
+            media_item: Union[Image, VideoFrame],
+            annotation_scene: Optional[AnnotationScene] = None,
     ) -> AnnotationScene:
         """
         Upload a new annotation for an image or video frame to the cluster. This will
@@ -257,7 +258,7 @@ def _upload_annotation_for_2d_media_item(
         return scene_to_upload
 
     def _append_annotation_for_2d_media_item(
-        self, media_item: Union[Image, VideoFrame]
+            self, media_item: Union[Image, VideoFrame]
     ) -> AnnotationScene:
         """
         Add an annotation to the existing annotations for the `media_item`.
@@ -299,9 +300,7 @@ def _append_annotation_for_2d_media_item(
         else:
             return annotation_scene
 
-    def _upload_annotations_for_2d_media_list(
-        self, media_list: Sequence[MediaItem], append_annotations: bool
-    ) -> int:
+    def _upload_annotations_for_2d_media_list(self, media_list: Sequence[MediaItem], append_annotations: bool) -> int:
         """
         Upload annotations to the server.
 
@@ -313,23 +312,52 @@ def _upload_annotations_for_2d_media_list(
         :return: Returns the number of uploaded annotations.
         """
         upload_count = 0
+        skip_count = 0
         tqdm_prefix = "Uploading media annotations"
-        with logging_redirect_tqdm(tqdm_class=tqdm):
-            for media_item in tqdm(media_list, desc=tqdm_prefix):
+
+        def upload_annotation(media_item: MediaItem) -> None:
+            nonlocal upload_count, skip_count
+            try:
                 if not append_annotations:
-                    response = self._upload_annotation_for_2d_media_item(
-                        media_item=media_item
+                    response = self._upload_annotation_for_2d_media_item(media_item=media_item)
+                else:
+                    response = self._append_annotation_for_2d_media_item(media_item=media_item)
+            except GetiRequestException as error:
+                skip_count += 1
+                if error.status_code == 500:
+                    logging.error(
+                        f"Failed to upload annotation for {media_item.name}. "
                     )
+                    return
                 else:
-                    response = self._append_annotation_for_2d_media_item(
-                        media_item=media_item
+                    raise error
+            if response is not None:
+                upload_count += 1
+
+        t_start = time.time()
+        with ThreadPoolExecutor(max_workers=5) as executor:
+            with logging_redirect_tqdm(tqdm_class=tqdm):
+                list(
+                    tqdm(
+                        executor.map(upload_annotation, media_list),
+                        total=len(media_list),
+                        desc=tqdm_prefix,
                     )
-                if response.annotations:
-                    upload_count += 1
+                )
+
+        t_elapsed = time.time() - t_start
+        if upload_count > 0:
+            logging.info(
+                f"Uploaded {upload_count} annotations in {t_elapsed:.1f} seconds."
+            )
+        if skip_count > 0:
+            logging.info(
+                f"Skipped {skip_count} media items, unable to upload annotations."
+            )
         return upload_count
 
     def annotation_scene_from_rest_response(
-        self, response_dict: Dict[str, Any], media_information: MediaInformation
+            self, response_dict: Dict[str, Any], media_information: MediaInformation
     ) -> AnnotationScene:
         """
         Convert a dictionary with annotation data obtained from the Intel® Geti™
@@ -344,7 +372,7 @@ def annotation_scene_from_rest_response(
         return annotation_scene
 
     def _get_latest_annotation_for_2d_media_item(
-        self, media_item: Union[Image, VideoFrame]
+            self, media_item: Union[Image, VideoFrame]
     ) -> Optional[AnnotationScene]:
         """
         Retrieve the latest annotation for an image or video frame from the cluster.
@@ -369,9 +397,9 @@ def _get_latest_annotation_for_2d_media_item(
         return annotation_scene
 
     def _read_2d_media_annotation_from_source(
-        self,
-        media_item: Union[Image, VideoFrame],
-        preserve_shape_for_global_labels: bool = False,
+            self,
+            media_item: Union[Image, VideoFrame],
+            preserve_shape_for_global_labels: bool = False,
     ) -> AnnotationScene:
         """
         Retrieve the annotation for the media_item, and return it in the
@@ -396,12 +424,12 @@ def _read_2d_media_annotation_from_source(
         )
 
     def _download_annotations_for_2d_media_list(
-        self,
-        media_list: Union[MediaList[Image], MediaList[VideoFrame]],
-        path_to_folder: str,
-        append_media_uid: bool = False,
-        verbose: bool = True,
-        max_threads: int = 10,
+            self,
+            media_list: Union[MediaList[Image], MediaList[VideoFrame]],
+            path_to_folder: str,
+            append_media_uid: bool = False,
+            verbose: bool = True,
+            max_threads: int = 10,
     ) -> float:
         """
         Download annotations from the server to a target folder on disk.
@@ -523,8 +551,8 @@ def download_annotation(media_item: Union[Image, VideoFrame]) -> None:
             msg = "No annotations were downloaded."
         if skip_count > 0:
             msg = (
-                msg + f" Was unable to retrieve annotations for {skip_count} "
-                f"{media_name_plural}, these {media_name_plural} were skipped."
+                    msg + f" Was unable to retrieve annotations for {skip_count} "
+                          f"{media_name_plural}, these {media_name_plural} were skipped."
             )
         if verbose:
             logging.info(msg)

From 001a25c11630818138dd82e9e6711670f5cf632d Mon Sep 17 00:00:00 2001
From: Daan Krol <daan.krol@intel.com>
Date: Wed, 23 Oct 2024 12:58:18 +0200
Subject: [PATCH 2/5] Formatting

---
 .../base_annotation_client.py                 | 69 ++++++++++---------
 1 file changed, 37 insertions(+), 32 deletions(-)

diff --git a/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py b/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
index 2c039442..a9d80fe2 100644
--- a/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
+++ b/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
@@ -36,10 +36,9 @@
 from geti_sdk.data_models.containers.media_list import MediaList
 from geti_sdk.data_models.label import Label
 from geti_sdk.data_models.media import MediaInformation, MediaItem
-from geti_sdk.http_session import GetiSession
+from geti_sdk.http_session import GetiRequestException, GetiSession
 from geti_sdk.rest_clients.dataset_client import DatasetClient
 from geti_sdk.rest_converters import AnnotationRESTConverter
-from geti_sdk.http_session import GetiRequestException
 
 AnnotationReaderType = TypeVar("AnnotationReaderType", bound=AnnotationReader)
 MediaType = TypeVar("MediaType", Image, Video)
@@ -51,11 +50,11 @@ class BaseAnnotationClient:
     """
 
     def __init__(
-            self,
-            session: GetiSession,
-            workspace_id: str,
-            project: Project,
-            annotation_reader: Optional[AnnotationReaderType] = None,
+        self,
+        session: GetiSession,
+        workspace_id: str,
+        project: Project,
+        annotation_reader: Optional[AnnotationReaderType] = None,
     ):
         self.session = session
         self.workspace_id = workspace_id
@@ -71,7 +70,7 @@ def __init__(
         )
 
     def _get_all_media_by_type(
-            self, media_type: Type[MediaType]
+        self, media_type: Type[MediaType]
     ) -> MediaList[MediaType]:
         """
         Get a list holding all media entities of type `media_type` in the project.
@@ -90,7 +89,7 @@ def _get_all_media_by_type(
         return media_list
 
     def _get_all_media_in_dataset_by_type(
-            self, media_type: Type[MediaType], dataset: Dataset
+        self, media_type: Type[MediaType], dataset: Dataset
     ) -> MediaList[MediaType]:
         """
         Return a list of all media items of type `media_type` in the dataset `dataset`
@@ -164,13 +163,13 @@ def __get_label_mapping(self, project: Project) -> Dict[str, str]:
         # We include the project label names in the mapping, as we want to ensure that
         # we can match the labels from the source to the project labels.
         for source_label_name in source_label_names.union(
-                project_label_name_to_label.keys()
+            project_label_name_to_label.keys()
         ):
             if source_label_name in project_label_name_to_label:
                 label_id = project_label_name_to_label[source_label_name].id
             elif (
-                    source_label_name.casefold() in project_label_name_to_label
-                    and project_label_name_to_label[source_label_name.casefold()].is_empty
+                source_label_name.casefold() in project_label_name_to_label
+                and project_label_name_to_label[source_label_name.casefold()].is_empty
             ):
                 label_id = project_label_name_to_label[source_label_name.casefold()].id
             else:
@@ -203,9 +202,9 @@ def label_mapping(self) -> Dict[str, str]:
             )
 
     def _upload_annotation_for_2d_media_item(
-            self,
-            media_item: Union[Image, VideoFrame],
-            annotation_scene: Optional[AnnotationScene] = None,
+        self,
+        media_item: Union[Image, VideoFrame],
+        annotation_scene: Optional[AnnotationScene] = None,
     ) -> AnnotationScene:
         """
         Upload a new annotation for an image or video frame to the cluster. This will
@@ -258,7 +257,7 @@ def _upload_annotation_for_2d_media_item(
         return scene_to_upload
 
     def _append_annotation_for_2d_media_item(
-            self, media_item: Union[Image, VideoFrame]
+        self, media_item: Union[Image, VideoFrame]
     ) -> AnnotationScene:
         """
         Add an annotation to the existing annotations for the `media_item`.
@@ -300,7 +299,9 @@ def _append_annotation_for_2d_media_item(
         else:
             return annotation_scene
 
-    def _upload_annotations_for_2d_media_list(self, media_list: Sequence[MediaItem], append_annotations: bool) -> int:
+    def _upload_annotations_for_2d_media_list(
+        self, media_list: Sequence[MediaItem], append_annotations: bool
+    ) -> int:
         """
         Upload annotations to the server.
 
@@ -319,9 +320,13 @@ def upload_annotation(media_item: MediaItem) -> None:
             nonlocal upload_count, skip_count
             try:
                 if not append_annotations:
-                    response = self._upload_annotation_for_2d_media_item(media_item=media_item)
+                    response = self._upload_annotation_for_2d_media_item(
+                        media_item=media_item
+                    )
                 else:
-                    response = self._append_annotation_for_2d_media_item(media_item=media_item)
+                    response = self._append_annotation_for_2d_media_item(
+                        media_item=media_item
+                    )
             except GetiRequestException as error:
                 skip_count += 1
                 if error.status_code == 500:
@@ -357,7 +362,7 @@ def upload_annotation(media_item: MediaItem) -> None:
         return upload_count
 
     def annotation_scene_from_rest_response(
-            self, response_dict: Dict[str, Any], media_information: MediaInformation
+        self, response_dict: Dict[str, Any], media_information: MediaInformation
     ) -> AnnotationScene:
         """
         Convert a dictionary with annotation data obtained from the Intel® Geti™
@@ -372,7 +377,7 @@ def annotation_scene_from_rest_response(
         return annotation_scene
 
     def _get_latest_annotation_for_2d_media_item(
-            self, media_item: Union[Image, VideoFrame]
+        self, media_item: Union[Image, VideoFrame]
     ) -> Optional[AnnotationScene]:
         """
         Retrieve the latest annotation for an image or video frame from the cluster.
@@ -397,9 +402,9 @@ def _get_latest_annotation_for_2d_media_item(
         return annotation_scene
 
     def _read_2d_media_annotation_from_source(
-            self,
-            media_item: Union[Image, VideoFrame],
-            preserve_shape_for_global_labels: bool = False,
+        self,
+        media_item: Union[Image, VideoFrame],
+        preserve_shape_for_global_labels: bool = False,
     ) -> AnnotationScene:
         """
         Retrieve the annotation for the media_item, and return it in the
@@ -424,12 +429,12 @@ def _read_2d_media_annotation_from_source(
         )
 
     def _download_annotations_for_2d_media_list(
-            self,
-            media_list: Union[MediaList[Image], MediaList[VideoFrame]],
-            path_to_folder: str,
-            append_media_uid: bool = False,
-            verbose: bool = True,
-            max_threads: int = 10,
+        self,
+        media_list: Union[MediaList[Image], MediaList[VideoFrame]],
+        path_to_folder: str,
+        append_media_uid: bool = False,
+        verbose: bool = True,
+        max_threads: int = 10,
     ) -> float:
         """
         Download annotations from the server to a target folder on disk.
@@ -551,8 +556,8 @@ def download_annotation(media_item: Union[Image, VideoFrame]) -> None:
             msg = "No annotations were downloaded."
         if skip_count > 0:
             msg = (
-                    msg + f" Was unable to retrieve annotations for {skip_count} "
-                          f"{media_name_plural}, these {media_name_plural} were skipped."
+                msg + f" Was unable to retrieve annotations for {skip_count} "
+                f"{media_name_plural}, these {media_name_plural} were skipped."
             )
         if verbose:
             logging.info(msg)

From 85d6068950aa91bd5a83c0780bd472d864380869 Mon Sep 17 00:00:00 2001
From: Daan Krol <daan.krol@intel.com>
Date: Wed, 30 Oct 2024 09:27:38 +0100
Subject: [PATCH 3/5] max_threads for annotation upload as parameter

---
 geti_sdk/geti.py                              | 10 +++-
 .../import_export/import_export_module.py     |  3 +-
 .../annotation_clients/annotation_client.py   | 56 +++++++++++++++----
 .../base_annotation_client.py                 | 13 ++++-
 4 files changed, 64 insertions(+), 18 deletions(-)

diff --git a/geti_sdk/geti.py b/geti_sdk/geti.py
index 8c017c9c..445d0fbe 100644
--- a/geti_sdk/geti.py
+++ b/geti_sdk/geti.py
@@ -713,10 +713,12 @@ def create_single_task_project_from_dataset(
             workspace_id=self.workspace_id,
             annotation_reader=annotation_reader,
         )
-        annotation_client.upload_annotations_for_images(images)
+        annotation_client.upload_annotations_for_images(images, max_threads=max_threads)
 
         if len(videos) > 0:
-            annotation_client.upload_annotations_for_videos(videos)
+            annotation_client.upload_annotations_for_videos(
+                videos, max_threads=max_threads
+            )
 
         configuration_client.set_project_auto_train(auto_train=enable_auto_train)
         return project
@@ -854,7 +856,9 @@ def create_task_chain_project_from_dataset(
                     annotation_reader=reader,
                 )
                 annotation_client.upload_annotations_for_images(
-                    images=images, append_annotations=append_annotations
+                    images=images,
+                    append_annotations=append_annotations,
+                    max_threads=max_threads,
                 )
                 append_annotations = True
             previous_task_type = task_type
diff --git a/geti_sdk/import_export/import_export_module.py b/geti_sdk/import_export/import_export_module.py
index f6d1ff2d..aa491ecd 100644
--- a/geti_sdk/import_export/import_export_module.py
+++ b/geti_sdk/import_export/import_export_module.py
@@ -238,6 +238,7 @@ def upload_project_data(
         if len(images) > 0:
             annotation_client.upload_annotations_for_images(
                 images=images,
+                max_threads=max_threads,
             )
         if len(videos) > 0:
             are_videos_processed = False
@@ -253,7 +254,7 @@ def upload_project_data(
                 are_videos_processed = uploaded_ids.issubset(project_video_ids)
                 time.sleep(1)
             annotation_client.upload_annotations_for_videos(
-                videos=videos,
+                videos=videos, max_threads=max_threads
             )
 
         configuration_file = os.path.join(target_folder, "configuration.json")
diff --git a/geti_sdk/rest_clients/annotation_clients/annotation_client.py b/geti_sdk/rest_clients/annotation_clients/annotation_client.py
index 9d70e19e..4fbeb435 100644
--- a/geti_sdk/rest_clients/annotation_clients/annotation_client.py
+++ b/geti_sdk/rest_clients/annotation_clients/annotation_client.py
@@ -58,7 +58,7 @@ def get_latest_annotations_for_video(self, video: Video) -> List[AnnotationScene
         return annotation_scenes
 
     def upload_annotations_for_video(
-        self, video: Video, append_annotations: bool = False
+        self, video: Video, append_annotations: bool = False, max_threads: int = 5
     ):
         """
         Upload annotations for a video. If append_annotations is set to True,
@@ -66,7 +66,11 @@ def upload_annotations_for_video(
         project. If set to False, existing annotations will be overwritten.
 
         :param video: Video to upload annotations for
-        :param append_annotations:
+        :param append_annotations: True to append annotations from the local disk to
+            the existing annotations on the server, False to overwrite the server
+            annotations by those on the local disk.
+        :param max_threads: Maximum number of threads to use for uploading. Defaults to 5.
+            Set to -1 to use all available threads.
         :return:
         """
         annotation_filenames = self.annotation_reader.get_data_filenames()
@@ -83,12 +87,17 @@ def upload_annotations_for_video(
             ]
         )
         upload_count = self._upload_annotations_for_2d_media_list(
-            media_list=video_frames, append_annotations=append_annotations
+            media_list=video_frames,
+            append_annotations=append_annotations,
+            max_threads=max_threads,
         )
         return upload_count
 
     def upload_annotations_for_videos(
-        self, videos: Sequence[Video], append_annotations: bool = False
+        self,
+        videos: Sequence[Video],
+        append_annotations: bool = False,
+        max_threads: int = 5,
     ):
         """
         Upload annotations for a list of videos. If append_annotations is set to True,
@@ -96,14 +105,20 @@ def upload_annotations_for_videos(
         project. If set to False, existing annotations will be overwritten.
 
         :param videos: List of videos to upload annotations for
-        :param append_annotations:
+        :param append_annotations: True to append annotations from the local disk to
+            the existing annotations on the server, False to overwrite the server
+            annotations by those on the local disk.
+        :param max_threads: Maximum number of threads to use for uploading. Defaults to 5.
+            Set to -1 to use all available threads.
         :return:
         """
         logging.info("Starting video annotation upload...")
         upload_count = 0
         for video in videos:
             upload_count += self.upload_annotations_for_video(
-                video=video, append_annotations=append_annotations
+                video=video,
+                append_annotations=append_annotations,
+                max_threads=max_threads,
             )
         if upload_count > 0:
             logging.info(
@@ -113,7 +128,10 @@ def upload_annotations_for_videos(
             logging.info("No new video frame annotations were found.")
 
     def upload_annotations_for_images(
-        self, images: Sequence[Image], append_annotations: bool = False
+        self,
+        images: Sequence[Image],
+        append_annotations: bool = False,
+        max_threads: int = 5,
     ):
         """
         Upload annotations for a list of images. If append_annotations is set to True,
@@ -121,12 +139,18 @@ def upload_annotations_for_images(
         project. If set to False, existing annotations will be overwritten.
 
         :param images: List of images to upload annotations for
-        :param append_annotations:
+        :param append_annotations: True to append annotations from the local disk to
+            the existing annotations on the server, False to overwrite the server
+            annotations by those on the local disk.
+        :param max_threads: Maximum number of threads to use for uploading. Defaults to 5.
+            Set to -1 to use all available threads.
         :return:
         """
         logging.info("Starting image annotation upload...")
         upload_count = self._upload_annotations_for_2d_media_list(
-            media_list=images, append_annotations=append_annotations
+            media_list=images,
+            append_annotations=append_annotations,
+            max_threads=max_threads,
         )
         if upload_count > 0:
             logging.info(
@@ -271,7 +295,9 @@ def download_all_annotations(
                 max_threads=max_threads,
             )
 
-    def upload_annotations_for_all_media(self, append_annotations: bool = False):
+    def upload_annotations_for_all_media(
+        self, append_annotations: bool = False, max_threads: int = 5
+    ):
         """
         Upload annotations for all media in the project, If append_annotations is set
         to True, annotations will be appended to the existing annotations for the
@@ -280,16 +306,22 @@ def upload_annotations_for_all_media(self, append_annotations: bool = False):
         :param append_annotations: True to append annotations from the local disk to
             the existing annotations on the server, False to overwrite the server
             annotations by those on the local disk. Defaults to False.
+        :param max_threads: Maximum number of threads to use for uploading. Defaults to 5.
+            Set to -1 to use all available threads.
         """
         image_list = self._get_all_media_by_type(media_type=Image)
         video_list = self._get_all_media_by_type(media_type=Video)
         if len(image_list) > 0:
             self.upload_annotations_for_images(
-                images=image_list, append_annotations=append_annotations
+                images=image_list,
+                append_annotations=append_annotations,
+                max_threads=max_threads,
             )
         if len(video_list) > 0:
             self.upload_annotations_for_videos(
-                videos=video_list, append_annotations=append_annotations
+                videos=video_list,
+                append_annotations=append_annotations,
+                max_threads=max_threads,
             )
 
     def upload_annotation(
diff --git a/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py b/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
index a9d80fe2..c2a50ee3 100644
--- a/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
+++ b/geti_sdk/rest_clients/annotation_clients/base_annotation_client.py
@@ -300,7 +300,10 @@ def _append_annotation_for_2d_media_item(
             return annotation_scene
 
     def _upload_annotations_for_2d_media_list(
-        self, media_list: Sequence[MediaItem], append_annotations: bool
+        self,
+        media_list: Sequence[MediaItem],
+        append_annotations: bool,
+        max_threads: int = 5,
     ) -> int:
         """
         Upload annotations to the server.
@@ -310,8 +313,14 @@ def _upload_annotations_for_2d_media_list(
         :param append_annotations: True to append annotations from the local disk to
             the existing annotations on the server, False to overwrite the server
             annotations by those on the local disk.
+        :param max_threads: Maximum number of threads to use for uploading. Defaults to 5.
+            Set to -1 to use all available threads.
         :return: Returns the number of uploaded annotations.
         """
+        if max_threads <= 0:
+            # ThreadPoolExecutor will use minimum 5 threads for 1 core cpu
+            # and maximum 32 threads for multi-core cpu.
+            max_threads = None
         upload_count = 0
         skip_count = 0
         tqdm_prefix = "Uploading media annotations"
@@ -340,7 +349,7 @@ def upload_annotation(media_item: MediaItem) -> None:
                 upload_count += 1
 
         t_start = time.time()
-        with ThreadPoolExecutor(max_workers=5) as executor:
+        with ThreadPoolExecutor(max_workers=max_threads) as executor:
             with logging_redirect_tqdm(tqdm_class=tqdm):
                 list(
                     tqdm(

From 507591c99ad7aa988db5a3ba5babb3d03816af85 Mon Sep 17 00:00:00 2001
From: Daan Krol <daan.krol@intel.com>
Date: Wed, 20 Nov 2024 17:12:20 +0100
Subject: [PATCH 4/5] limit annotation upload threads in tests

---
 tests/helpers/project_service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/helpers/project_service.py b/tests/helpers/project_service.py
index 0630aca2..680e476b 100644
--- a/tests/helpers/project_service.py
+++ b/tests/helpers/project_service.py
@@ -435,7 +435,7 @@ def add_annotated_media(
                 ]
                 # Upload annotations
                 self.annotation_client.upload_annotations_for_images(
-                    images=images, append_annotations=task_index > 0
+                    images=images, append_annotations=task_index > 0, max_threads=1
                 )
 
     def set_auto_train(self, auto_train: bool = True) -> None:

From 4620fd31dca78dc104fb87bc1ea4d188bab2e8bf Mon Sep 17 00:00:00 2001
From: Daan Krol <daan.krol@intel.com>
Date: Wed, 20 Nov 2024 17:20:27 +0100
Subject: [PATCH 5/5] limit video annotation upload threads in tests

---
 .../integration/rest_clients/test_annotation_client.py      | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/pre-merge/integration/rest_clients/test_annotation_client.py b/tests/pre-merge/integration/rest_clients/test_annotation_client.py
index aac2d5df..86cf5b1f 100644
--- a/tests/pre-merge/integration/rest_clients/test_annotation_client.py
+++ b/tests/pre-merge/integration/rest_clients/test_annotation_client.py
@@ -88,7 +88,7 @@ def test_upload_and_retrieve_annotations_for_video(
         if fxt_test_mode != SdkTestMode.OFFLINE:
             time.sleep(1)
 
-        annotation_client.upload_annotations_for_video(video=video)
+        annotation_client.upload_annotations_for_video(video=video, max_threads=1)
 
         if fxt_test_mode != SdkTestMode.OFFLINE:
             time.sleep(1)
@@ -198,7 +198,9 @@ def test_upload_and_retrieve_annotations_for_videos(
         annotation_client = fxt_project_service.annotation_client
         annotation_client.annotation_reader = annotation_reader
 
-        annotation_client.upload_annotations_for_videos(videos=[video_1, video_2])
+        annotation_client.upload_annotations_for_videos(
+            videos=[video_1, video_2], max_threads=1
+        )
 
         if fxt_test_mode != SdkTestMode.OFFLINE:
             time.sleep(10)