Notification for embedder warnings (#5)

* adds notification for embedder warnings

* adds embedders directory to gitignore

* prints last embedder warning in notification

* sends embedding warning as notification of type warning

* adds primary key or record_id to embedding warning

* fixes spelling

* updates embedders version number

Co-authored-by: JWittmeyer <[email protected]>

Author: FelixKirsch

.gitignore

@@ -129,4 +129,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-.DS_Store
+.DS_Store
+
+# embedders package
+embedders/

controller.py

@@ -8,6 +8,7 @@
     record,
     tokenization,
     notification,
+    organization,
 )
 import torch
 import traceback
@@ -24,10 +25,9 @@
 from util import daemon, request_util
 from util.decorator import param_throttle
 from util.embedders import get_embedder
-from util.notification import send_project_update
+from util.notification import send_project_update, embedding_warning_templates
 import os
 import pandas as pd
-from submodules.model.business_objects import embedding, general, organization
 from submodules.s3 import controller as s3
 
 logging.basicConfig(level=logging.INFO)
@@ -133,7 +133,7 @@ def prepare_run_encoding(request: data_type.Request, embedding_type: str) -> int
                         request.project_id,
                         request.user_id,
                         message,
-                        "ERROR",
+                        enums.Notification.ERROR.value,
                         enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
                         True,
                     )
@@ -142,11 +142,6 @@ def prepare_run_encoding(request: data_type.Request, embedding_type: str) -> int
                         f"notification_created:{request.user_id}",
                         True,
                     )
-                    embedding.update_embedding_state_failed(
-                        request.project_id,
-                        embedding_id,
-                        with_commit=True,
-                    )
                     doc_ock.post_embedding_failed(
                         request.user_id, request.config_string
                     )
@@ -175,7 +170,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Initializing model {request.config_string}. This can take a few minutes.",
-            "INFO",
+            enums.Notification.INFO.value,
             enums.NotificationType.EMBEDDING_CREATION_STARTED.value,
             True,
         )
@@ -184,7 +179,9 @@ def run_encoding(
         )
         iso2_code = project.get_blank_tokenizer_from_project(request.project_id)
         try:
-            embedder = get_embedder(request.project_id, embedding_type, request.config_string, iso2_code)
+            embedder = get_embedder(
+                request.project_id, embedding_type, request.config_string, iso2_code
+            )
         except OSError:
             embedding.update_embedding_state_failed(
                 request.project_id,
@@ -201,7 +198,7 @@ def run_encoding(
                 request.project_id,
                 request.user_id,
                 message,
-                "ERROR",
+                enums.Notification.ERROR.value,
                 enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
                 True,
             )
@@ -226,7 +223,7 @@ def run_encoding(
                 request.project_id,
                 request.user_id,
                 message,
-                "ERROR",
+                enums.Notification.ERROR.value,
                 enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
                 True,
             )
@@ -240,7 +237,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Could not load model {request.config_string}. Please contact the support.",
-            "ERROR",
+            enums.Notification.ERROR.value,
             enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
             True,
         )
@@ -283,7 +280,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Started encoding {attribute_name} using model {request.config_string}.",
-            "INFO",
+            enums.Notification.INFO.value,
             enums.NotificationType.EMBEDDING_CREATION_STARTED.value,
             True,
         )
@@ -324,6 +321,39 @@ def run_encoding(
                 initial_count,
             )
     except Exception:
+        for warning_type, idx_list in embedder.get_warnings().items():
+            # use last record with warning as example
+            example_record_id = record_ids[idx_list[-1]]
+
+            primary_keys = [
+                pk.name for pk in attribute.get_primary_keys(request.project_id)
+            ]
+            if primary_keys:
+                example_record_data = record.get(
+                    request.project_id, example_record_id
+                ).data
+                example_record_msg = "with primary key: " + ", ".join(
+                    [str(example_record_data[p_key]) for p_key in primary_keys]
+                )
+            else:
+                example_record_msg = " with record id: " + str(example_record_id)
+
+            warning_msg = embedding_warning_templates[warning_type].format(
+                record_number=len(idx_list), example_record_msg=example_record_msg
+            )
+
+            notification.create(
+                request.project_id,
+                request.user_id,
+                warning_msg,
+                enums.Notification.WARNING.value,
+                enums.NotificationType.EMBEDDING_CREATION_WARNING.value,
+                True,
+            )
+            send_project_update(
+                request.project_id, f"notification_created:{request.user_id}", True
+            )
+
         embedding.update_embedding_state_failed(
             request.project_id,
             embedding_id,
@@ -337,27 +367,51 @@ def run_encoding(
             request.project_id,
             request.user_id,
             "Error at runtime. Please contact support.",
-            "ERROR",
+            enums.Notification.ERROR.value,
             enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
             True,
         )
         send_project_update(
             request.project_id, f"notification_created:{request.user_id}", True
         )
         print(traceback.format_exc(), flush=True)
-        embedding.update_embedding_state_failed(
-            request.project_id,
-            embedding_id,
-            with_commit=True,
-        )
-        send_project_update(
-            request.project_id,
-            f"embedding:{embedding_id}:state:{enums.EmbeddingState.FAILED.value}",
-        )
         doc_ock.post_embedding_failed(request.user_id, request.config_string)
         return 500
 
     if embedding.get(request.project_id, embedding_id):
+        for warning_type, idx_list in embedder.get_warnings().items():
+            # use last record with warning as example
+            example_record_id = record_ids[idx_list[-1]]
+
+            primary_keys = [
+                pk.name for pk in attribute.get_primary_keys(request.project_id)
+            ]
+            if primary_keys:
+                example_record_data = record.get(
+                    request.project_id, example_record_id
+                ).data
+                example_record_msg = "with primary key: " + ", ".join(
+                    [str(example_record_data[p_key]) for p_key in primary_keys]
+                )
+            else:
+                example_record_msg = " with record id: " + str(example_record_id)
+
+            warning_msg = embedding_warning_templates[warning_type].format(
+                record_number=len(idx_list), example_record_msg=example_record_msg
+            )
+
+            notification.create(
+                request.project_id,
+                request.user_id,
+                warning_msg,
+                enums.Notification.WARNING.value,
+                enums.NotificationType.EMBEDDING_CREATION_WARNING.value,
+                True,
+            )
+            send_project_update(
+                request.project_id, f"notification_created:{request.user_id}", True
+            )
+
         if embedding_type == "classification":
             request_util.post_embedding_to_neural_search(
                 request.project_id, embedding_id
@@ -376,7 +430,7 @@ def run_encoding(
             request.project_id,
             request.user_id,
             f"Finished encoding {attribute_name} using model {request.config_string}.",
-            "SUCCESS",
+            enums.Notification.SUCCESS.value,
             enums.NotificationType.EMBEDDING_CREATION_DONE.value,
             True,
         )

requirements.txt

@@ -7,7 +7,7 @@ certifi==2021.10.8
 charset-normalizer==2.0.12
 click==8.0.4
 cymem==2.0.6
-embedders==0.0.14
+embedders==0.0.15
 fastapi==0.78.0
 greenlet==1.1.2
 h11==0.13.0

submodules/model

@@ -1,8 +1,23 @@
 import requests
 import os
 
+from embedders.enums import WarningType
+
 from submodules.model.business_objects import project
 
+embedding_warning_templates = {
+    WarningType.DOCUMENT_IS_SPLITTED.value: (
+        "For {record_number} records, the text length exceeds the model's max input"
+        " length. For these records, the texts are splitted and the parts are processed"
+        " individually. For example, record {example_record_msg}."
+    ),
+    WarningType.TOKEN_MISMATCHING.value: (
+        "For {record_number} records, the number of embeddings does not match the "
+        "number of spacy tokens. Please contact support. For example, record "
+        "{example_record_msg}."
+    ),
+}
+
 
 def send_project_update(project_id: str, message: str, is_global: bool = False) -> None:
     endpoint = os.getenv("WS_NOTIFY_ENDPOINT")