feat: mlserver-mlflow integration (#161)

#133 Integration tests for mlsever-mlflow. Similar to other mlserver-* tests. Summary of changes: - Added integration tests files. - Updated tests. - Updated config map to point to published rock.
canonical · Jul 10, 2023 · 59b48cf · 59b48cf
1 parent 77c3f6d
commit 59b48cf
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 39 deletions.
diff --git a/src/templates/configmap.yaml.j2 b/src/templates/configmap.yaml.j2
@@ -60,8 +60,8 @@ data:
               "defaultImageVersion": "1.15.0"
               },
             "v2": {
-              "image": "seldonio/mlserver",
-              "defaultImageVersion": "1.2.0-mlflow"
+              "image": "docker.io/charmedkubeflow/mlserver-mlflow_1.2.0_22.04_1_amd64",
+              "defaultImageVersion": "1.2.0_22.04_1"
               }
             }
         },

diff --git a/tests/assets/crs/mlflowserver-v2.yaml b/tests/assets/crs/mlflowserver-v2.yaml
@@ -17,7 +17,7 @@ spec:
     - graph:
         children: []
         implementation: MLFLOW_SERVER
-        modelUri: gs://seldon-models/mlflow/elasticnet_wine_1.8.0
+        modelUri: gs://seldon-models/v1.12.0-dev/mlflow/elasticnet_wine
         name: classifier
       name: default
       replicas: 1
diff --git a/tests/assets/data/mlflowserver-request-data.json b/tests/assets/data/mlflowserver-request-data.json
@@ -0,0 +1,82 @@
+{
+    "parameters": {"content_type": "pd"},
+    "inputs": [
+        {
+            "name": "fixed acidity",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [7.4],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "volatile acidity",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [0.7000],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "citric acidity",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [0],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "residual sugar",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [1.9],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "chlorides",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [0.076],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "free sulfur dioxide",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [11],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "total sulfur dioxide",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [34],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "density",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [0.9978],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "pH",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [3.51],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "sulphates",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [0.56],
+            "parameters": {"content_type": "np"}
+        },
+        {
+            "name": "alcohol",
+            "shape": [1],
+            "datatype": "FP32",
+            "data": [9.4],
+            "parameters": {"content_type": "np"}
+        }
+    ]
+}
diff --git a/tests/assets/data/mlflowserver-response-data.json b/tests/assets/data/mlflowserver-response-data.json
@@ -0,0 +1,24 @@
+{
+    "model_name": "classifier",
+    "model_version": "v1",
+    "id": "None",
+    "parameters": {
+        "content_type": "np"
+    },
+    "outputs": [
+        {
+            "name": "output-1",
+            "shape": [
+                1,
+                1
+            ],
+            "datatype": "FP64",
+            "parameters": {
+                "content_type": "np"
+            },
+            "data": [
+                6.016145744177844
+            ]
+        }
+    ]
+}
diff --git a/tests/integration/test_seldon_servers.py b/tests/integration/test_seldon_servers.py
@@ -96,7 +96,7 @@ async def test_build_and_deploy(ops_test: OpsTest):
             {
                 "model_name": "classifier",
                 "model_version": "v1",
-                "id": None,  # id needs to be reset in response
+                "id": "None",  # id needs to be reset in response
                 "parameters": {"content_type": None, "headers": None},
                 "outputs": [
                     {
@@ -140,7 +140,7 @@ async def test_build_and_deploy(ops_test: OpsTest):
             {
                 "model_name": "iris",
                 "model_version": "v0.1.0",
-                "id": None,  # id needs to be reset in response
+                "id": "None",  # id needs to be reset in response
                 "parameters": {"content_type": None, "headers": None},
                 "outputs": [
                     {
@@ -167,37 +167,13 @@ async def test_build_and_deploy(ops_test: OpsTest):
                 "meta": {"requestPath": {"classifier": "IMAGE:VERSION"}},
             },
         ),
-        # Disable test for mlflowserver V2 due to failure in model in test container
-        # (
-        #     "MLFLOW_SERVER",
-        #     "mlflowserver-v2.yaml",
-        #     "v2/models/iris/infer",
-        #     {
-        #         "inputs": [
-        #             {
-        #                 "name": "predict",
-        #                 "shape": [1, 4],
-        #                 "datatype": "FP32",
-        #                 "data": [[1, 2, 3, 4]],
-        #             },
-        #         ]
-        #     },
-        #     {
-        #         "model_name": "iris",
-        #         "model_version": "v0.1.0",
-        #         "id": None,  # id needs to be reset in response
-        #         "parameters": {"content_type": None, "headers": None},
-        #         "outputs": [
-        #             {
-        #                 "name": "predict",
-        #                 "shape": [1, 1],
-        #                 "datatype": "FP32",
-        #                 "parameters": None,
-        #                 "data": [2.0],
-        #             }
-        #         ],
-        #     },
-        # ),
+        (
+            "MLFLOW_SERVER",
+            "mlflowserver-v2.yaml",
+            "v2/models/classifier/infer",
+            "mlflowserver-request-data.json",
+            "mlflowserver-response-data.json",
+        ),
         (
             "TENSORFLOW_SERVER",
             "tensorflow-serving.yaml",
@@ -243,15 +219,15 @@ async def test_seldon_predictor_server(
         client.create(sdep, namespace=namespace)
 
     # prepare request data:
-    # - if it is string, load it from file specified in by that string
+    # - if it is string, load it from file specified by that string
     # - otherwise use it as JSON object
     if isinstance(request_data, str):
         # response test data contains file with JSON data
         with open(f"tests/assets/data/{request_data}") as f:
             request_data = json.load(f)
 
     # prepare test response data:
-    # - if it is string, load it from file specified in by that string
+    # - if it is string, load it from file specified by that string
     # - otherwise use it as JSON object
     if isinstance(response_test_data, str):
         # response test data contains file with JSON data
@@ -274,7 +250,7 @@ async def test_seldon_predictor_server(
 
     # reset id in response, if present
     if "id" in response.keys():
-        response["id"] = None
+        response["id"] = "None"
 
     # for 'seldon' protocol update test data with correct predictor server image
     if protocol == "seldon":