integrates model-provider service (#9)

* integrates model-provider service

* adds project_id to model-provider request

* removes project_id and revision from model-provider api calls

* try auto select model dir

* adds error message if model is deleted during creation

Co-authored-by: JWittmeyer <[email protected]>

Author: FelixKirsch

check_config_service

@@ -0,0 +1,31 @@
+#!/bin/bash
+CONFIG_EXISTS=0
+if [ -n "$(docker ps -q -f name=dev-setup_refinery-config_1)" ]
+then
+    if [ "$( docker container inspect -f '{{.State.Status}}' dev-setup_refinery-config_1 )" == "running" ]; 
+    then
+        CONFIG_EXISTS=1
+    fi
+elif [ -n "$(docker ps -q -f name=dev-setup-refinery-config-1)" ]
+then
+    if [ "$( docker container inspect -f '{{.State.Status}}' dev-setup-refinery-config-1 )" == "running" ]; 
+    then
+        CONFIG_EXISTS=1
+    fi
+else
+    if [ -n "$(docker ps -q -f name=refinery-config)" ];
+    then
+        if [ "$( docker container inspect -f '{{.State.Status}}' refinery-config )" == "running" ];
+        then
+            CONFIG_EXISTS=1
+        fi
+    fi
+fi
+if [ $CONFIG_EXISTS -eq 0 ]
+then 
+    echo "refinery-config couldn't be found - exit"
+    exit 1
+else
+    echo "refinery-config found -> proceeding"
+fi
+

controller.py

@@ -23,6 +23,7 @@
 from typing import Any, Dict, Iterator, List
 
 from util import daemon, request_util
+from util.config_handler import get_config_value
 from util.decorator import param_throttle
 from util.embedders import get_embedder
 from util.notification import send_project_update, embedding_warning_templates
@@ -179,8 +180,15 @@ def run_encoding(
         )
         iso2_code = project.get_blank_tokenizer_from_project(request.project_id)
         try:
+            if not __is_embedders_internal_model(
+                request.config_string
+            ) and get_config_value("is_managed"):
+                config_string = request_util.get_model_path(request.config_string)
+            else:
+                config_string = request.config_string
+
             embedder = get_embedder(
-                request.project_id, embedding_type, request.config_string, iso2_code
+                request.project_id, embedding_type, config_string, iso2_code
             )
         except OSError:
             embedding.update_embedding_state_failed(
@@ -206,6 +214,30 @@ def run_encoding(
                 request.project_id, f"notification_created:{request.user_id}", True
             )
             return 422
+        except ValueError:
+            embedding.update_embedding_state_failed(
+                request.project_id,
+                embedding_id,
+                with_commit=True,
+            )
+            send_project_update(
+                request.project_id,
+                f"embedding:{embedding_id}:state:{enums.EmbeddingState.FAILED.value}",
+            )
+            doc_ock.post_embedding_failed(request.user_id, request.config_string)
+            message = f"Model {request.config_string} was deleted during the creation process."
+            notification.create(
+                request.project_id,
+                request.user_id,
+                message,
+                enums.Notification.ERROR.value,
+                enums.NotificationType.EMBEDDING_CREATION_FAILED.value,
+                True,
+            )
+            send_project_update(
+                request.project_id, f"notification_created:{request.user_id}", True
+            )
+            return 422
 
         if not embedder:
             embedding.update_embedding_state_failed(
@@ -503,3 +535,7 @@ def upload_embedding_as_file(
         sql_df.to_csv(file_name, mode="a", index=False)
     s3.upload_object(org_id, s3_file_name, file_name)
     os.remove(file_name)
+
+
+def __is_embedders_internal_model(model_name: str):
+    return model_name in ["bag-of-characters", "bag-of-words", "tf-idf"]

start

@@ -1,6 +1,8 @@
 #!/bin/bash
 trap "echo -ne '\nstopping container...' && docker stop refinery-embedder > /dev/null 2>&1 && echo -ne '\t\t [done]\n'" EXIT
 
+source check_config_service
+
 HOST_IP=$(docker network inspect bridge --format='{{json .IPAM.Config}}' | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | tail -1)
 
 echo -ne 'stopping old container...'
@@ -11,6 +13,20 @@ echo -ne 'building container...'
 docker build -t refinery-embedder-dev -f dev.Dockerfile .
 echo -ne '\t\t [done]\n'
 
+MODEL_DIR=${PWD%/*}/dev-setup/model-data/
+if [ ! -d "$MODEL_DIR" ] 
+then
+    MODEL_DIR=${PWD%/*/*}/dev-setup/model-data/
+    if [ ! -d "$MODEL_DIR" ] 
+    then
+        # to include volume for local development, use the dev-setup model data folder:  
+        # alternative use manual logic with
+        # -v /path/to/dev-setup/model-data:/models \
+        echo "Can't find model data directory: $MODEL_DIR -> stopping" 
+        exit 1
+    fi
+fi
+
 echo -ne 'starting...'
 docker run -d --rm \
 --name refinery-embedder \
@@ -20,12 +36,15 @@ docker run -d --rm \
 -e S3_SECRET_KEY=r6ywtR33!DMlaL*SUUdy \
 -e POSTGRES=postgresql://postgres:onetask@graphql-postgres:5432 \
 -e DOC_OCK=http://refinery-doc-ock:80 \
+-e MODEL_PROVIDER=http://refinery-model-provider:80 \
 -e WS_NOTIFY_ENDPOINT="http://refinery-websocket:8080" \
 -e NEURAL_SEARCH=http://refinery-neural-search:80 \
 --mount type=bind,source="$(pwd)"/,target=/app \
 -v /var/run/docker.sock:/var/run/docker.sock \
+-v "$MODEL_DIR":/models \
 --network dev-setup_default \
 refinery-embedder-dev > /dev/null 2>&1
 echo -ne '\t\t\t [done]\n'
 
+
 docker logs -f refinery-embedder

util/config_handler.py

@@ -0,0 +1,53 @@
+from typing import Dict, Any, Optional, Union
+import requests
+import json
+import time
+from util import daemon
+
+__config = None
+
+# meant as a const value since env variables will be removed at some point
+REQUEST_URL = "http://refinery-config:80/full_config"
+
+
+def __get_config() -> Dict[str, Any]:
+    global __config
+    if __config:
+        return __config
+    refresh_config()
+    return __config
+
+
+def refresh_config():
+    response = requests.get(REQUEST_URL)
+    if response.status_code == 200:
+        global __config
+        __config = json.loads(json.loads(response.text))
+        daemon.run(invalidate_after, 3600)  # one hour
+    else:
+        raise Exception(
+            f"Config service cant be reached -- response.code{response.status_code}"
+        )
+
+
+def get_config_value(
+    key: str, subkey: Optional[str] = None
+) -> Union[str, Dict[str, str]]:
+    config = __get_config()
+    if key not in config:
+        raise Exception(f"Key {key} coudn't be found in config")
+    value = config[key]
+
+    if not subkey:
+        return value
+
+    if isinstance(value, dict) and subkey in value:
+        return value[subkey]
+    else:
+        raise Exception(f"Subkey {subkey} coudn't be found in config[{key}]")
+
+
+def invalidate_after(sec: int) -> None:
+    time.sleep(sec)
+    global __config
+    __config = None

util/request_util.py

@@ -2,6 +2,7 @@
 import requests
 
 NEURAL_SEARCH_BASE_URI = os.getenv("NEURAL_SEARCH")
+MODEL_PROVIDER_BASE_URI = os.getenv("MODEL_PROVIDER")
 
 
 def post_embedding_to_neural_search(project_id: str, embedding_id: str) -> None:
@@ -17,3 +18,12 @@ def delete_embedding_from_neural_search(embedding_id: str) -> None:
     url = f"{NEURAL_SEARCH_BASE_URI}/delete_collection"
     params = {"embedding_id": embedding_id}
     requests.put(url, params=params)
+
+
+def get_model_path(model_name: str) -> str:
+    url = f"{MODEL_PROVIDER_BASE_URI}/model_path"
+    params = {
+        "model_name": model_name,
+    }
+    response = requests.get(url, params=params)
+    return response.json()