Skip to content

Commit

Permalink
feat: mlserver-mlflow integration (#161)
Browse files Browse the repository at this point in the history
#133

Integration tests for mlsever-mlflow. Similar to other mlserver-* tests.

Summary of changes:
- Added integration tests files.
- Updated tests.
- Updated config map to point to published rock.
  • Loading branch information
i-chvets authored Jul 10, 2023
1 parent 77c3f6d commit 59b48cf
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 39 deletions.
4 changes: 2 additions & 2 deletions src/templates/configmap.yaml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ data:
"defaultImageVersion": "1.15.0"
},
"v2": {
"image": "seldonio/mlserver",
"defaultImageVersion": "1.2.0-mlflow"
"image": "docker.io/charmedkubeflow/mlserver-mlflow_1.2.0_22.04_1_amd64",
"defaultImageVersion": "1.2.0_22.04_1"
}
}
},
Expand Down
2 changes: 1 addition & 1 deletion tests/assets/crs/mlflowserver-v2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ spec:
- graph:
children: []
implementation: MLFLOW_SERVER
modelUri: gs://seldon-models/mlflow/elasticnet_wine_1.8.0
modelUri: gs://seldon-models/v1.12.0-dev/mlflow/elasticnet_wine
name: classifier
name: default
replicas: 1
82 changes: 82 additions & 0 deletions tests/assets/data/mlflowserver-request-data.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"parameters": {"content_type": "pd"},
"inputs": [
{
"name": "fixed acidity",
"shape": [1],
"datatype": "FP32",
"data": [7.4],
"parameters": {"content_type": "np"}
},
{
"name": "volatile acidity",
"shape": [1],
"datatype": "FP32",
"data": [0.7000],
"parameters": {"content_type": "np"}
},
{
"name": "citric acidity",
"shape": [1],
"datatype": "FP32",
"data": [0],
"parameters": {"content_type": "np"}
},
{
"name": "residual sugar",
"shape": [1],
"datatype": "FP32",
"data": [1.9],
"parameters": {"content_type": "np"}
},
{
"name": "chlorides",
"shape": [1],
"datatype": "FP32",
"data": [0.076],
"parameters": {"content_type": "np"}
},
{
"name": "free sulfur dioxide",
"shape": [1],
"datatype": "FP32",
"data": [11],
"parameters": {"content_type": "np"}
},
{
"name": "total sulfur dioxide",
"shape": [1],
"datatype": "FP32",
"data": [34],
"parameters": {"content_type": "np"}
},
{
"name": "density",
"shape": [1],
"datatype": "FP32",
"data": [0.9978],
"parameters": {"content_type": "np"}
},
{
"name": "pH",
"shape": [1],
"datatype": "FP32",
"data": [3.51],
"parameters": {"content_type": "np"}
},
{
"name": "sulphates",
"shape": [1],
"datatype": "FP32",
"data": [0.56],
"parameters": {"content_type": "np"}
},
{
"name": "alcohol",
"shape": [1],
"datatype": "FP32",
"data": [9.4],
"parameters": {"content_type": "np"}
}
]
}
24 changes: 24 additions & 0 deletions tests/assets/data/mlflowserver-response-data.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"model_name": "classifier",
"model_version": "v1",
"id": "None",
"parameters": {
"content_type": "np"
},
"outputs": [
{
"name": "output-1",
"shape": [
1,
1
],
"datatype": "FP64",
"parameters": {
"content_type": "np"
},
"data": [
6.016145744177844
]
}
]
}
48 changes: 12 additions & 36 deletions tests/integration/test_seldon_servers.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ async def test_build_and_deploy(ops_test: OpsTest):
{
"model_name": "classifier",
"model_version": "v1",
"id": None, # id needs to be reset in response
"id": "None", # id needs to be reset in response
"parameters": {"content_type": None, "headers": None},
"outputs": [
{
Expand Down Expand Up @@ -140,7 +140,7 @@ async def test_build_and_deploy(ops_test: OpsTest):
{
"model_name": "iris",
"model_version": "v0.1.0",
"id": None, # id needs to be reset in response
"id": "None", # id needs to be reset in response
"parameters": {"content_type": None, "headers": None},
"outputs": [
{
Expand All @@ -167,37 +167,13 @@ async def test_build_and_deploy(ops_test: OpsTest):
"meta": {"requestPath": {"classifier": "IMAGE:VERSION"}},
},
),
# Disable test for mlflowserver V2 due to failure in model in test container
# (
# "MLFLOW_SERVER",
# "mlflowserver-v2.yaml",
# "v2/models/iris/infer",
# {
# "inputs": [
# {
# "name": "predict",
# "shape": [1, 4],
# "datatype": "FP32",
# "data": [[1, 2, 3, 4]],
# },
# ]
# },
# {
# "model_name": "iris",
# "model_version": "v0.1.0",
# "id": None, # id needs to be reset in response
# "parameters": {"content_type": None, "headers": None},
# "outputs": [
# {
# "name": "predict",
# "shape": [1, 1],
# "datatype": "FP32",
# "parameters": None,
# "data": [2.0],
# }
# ],
# },
# ),
(
"MLFLOW_SERVER",
"mlflowserver-v2.yaml",
"v2/models/classifier/infer",
"mlflowserver-request-data.json",
"mlflowserver-response-data.json",
),
(
"TENSORFLOW_SERVER",
"tensorflow-serving.yaml",
Expand Down Expand Up @@ -243,15 +219,15 @@ async def test_seldon_predictor_server(
client.create(sdep, namespace=namespace)

# prepare request data:
# - if it is string, load it from file specified in by that string
# - if it is string, load it from file specified by that string
# - otherwise use it as JSON object
if isinstance(request_data, str):
# response test data contains file with JSON data
with open(f"tests/assets/data/{request_data}") as f:
request_data = json.load(f)

# prepare test response data:
# - if it is string, load it from file specified in by that string
# - if it is string, load it from file specified by that string
# - otherwise use it as JSON object
if isinstance(response_test_data, str):
# response test data contains file with JSON data
Expand All @@ -274,7 +250,7 @@ async def test_seldon_predictor_server(

# reset id in response, if present
if "id" in response.keys():
response["id"] = None
response["id"] = "None"

# for 'seldon' protocol update test data with correct predictor server image
if protocol == "seldon":
Expand Down

0 comments on commit 59b48cf

Please sign in to comment.