Batch Endpoints scenarios (Azure#1777)

* batch scenarios

remove model

* black

* comments

* typo

* typos

* formatting

* build

* batch.sh fix

Author: santiagxf

.github/workflows/sdk-endpoints-batch-mnist-nonmlflow.yml

@@ -37,14 +37,14 @@ jobs:
       run: bash setup.sh
       working-directory: cli
       continue-on-error: true
-    - name: run endpoints/batch/mnist-nonmlflow.ipynb
+    - name: run endpoints/batch/mnist-batch.ipynb
       run: |
-          sed -i -e "s/<SUBSCRIPTION_ID>/6560575d-fa06-4e7d-95fb-f962e74efd7a/g" mnist-nonmlflow.ipynb
-          sed -i -e "s/<RESOURCE_GROUP>/azureml-examples/g" mnist-nonmlflow.ipynb
-          sed -i -e "s/<AML_WORKSPACE_NAME>/main/g" mnist-nonmlflow.ipynb
-          sed -i -e "s/DefaultAzureCredential/AzureCliCredential/g" mnist-nonmlflow.ipynb
+          sed -i -e "s/<SUBSCRIPTION_ID>/6560575d-fa06-4e7d-95fb-f962e74efd7a/g" mnist-batch.ipynb
+          sed -i -e "s/<RESOURCE_GROUP>/azureml-examples/g" mnist-batch.ipynb
+          sed -i -e "s/<AML_WORKSPACE_NAME>/main/g" mnist-batch.ipynb
+          sed -i -e "s/DefaultAzureCredential/AzureCliCredential/g" mnist-batch.ipynb
 
-          papermill -k python mnist-nonmlflow.ipynb mnist-nonmlflow.output.ipynb
+          papermill -k python mnist-batch.ipynb mnist-batch.output.ipynb
       working-directory: sdk/python/endpoints/batch
     - name: upload notebook's working folder as an artifact
       if: ${{ always() }}
@@ -63,11 +63,11 @@ jobs:
         private_key: ${{ secrets.AZUREML_ICM_CONNECTOR_PRIVATE_KEY }}
         args: |
             incident:
-                Title: "[azureml-examples] Notebook validation failed on branch '${{ github.ref_name }}' for notebook 'endpoints/batch/mnist-nonmlflow.ipynb'"
+                Title: "[azureml-examples] Notebook validation failed on branch '${{ github.ref_name }}' for notebook 'endpoints/batch/mnist-batch.ipynb'"
                 Summary: |
-                    Notebook 'endpoints/batch/mnist-nonmlflow.ipynb' is failing on branch '${{ github.ref_name }}': ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+                    Notebook 'endpoints/batch/mnist-batch.ipynb' is failing on branch '${{ github.ref_name }}': ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
                 Severity: 4
                 RoutingId: "github://azureml-examples"
                 Status: Active
                 Source:
-                    IncidentId: "endpoints/batch/mnist-nonmlflow.ipynb[${{ github.ref_name }}]"
+                    IncidentId: "endpoints/batch/mnist-batch.ipynb[${{ github.ref_name }}]"

cli/batch-score.sh

@@ -4,9 +4,11 @@ set -e
 
 # <set_variables>
 export ENDPOINT_NAME="<YOUR_ENDPOINT_NAME>"
+export DEPLOYMENT_NAME="<YOUR_DEPLOYMENT_NAME>"
 # </set_variables>
 
 export ENDPOINT_NAME=endpt-`echo $RANDOM`
+export DEPLOYMENT_NAME="mnist-torch-dpl"
 
 echo "Creating compute"
 # <create_compute>
@@ -20,7 +22,7 @@ az ml batch-endpoint create --name $ENDPOINT_NAME
 
 echo "Creating batch deployment nonmlflowdp for endpoint $ENDPOINT_NAME"
 # <create_batch_deployment_set_default>
-az ml batch-deployment create --name nonmlflowdp --endpoint-name $ENDPOINT_NAME --file endpoints/batch/nonmlflow-deployment.yml --set-default
+az ml batch-deployment create --file endpoints/batch/mnist-torch-deployment.yml --endpoint-name $ENDPOINT_NAME --set-default
 # </create_batch_deployment_set_default>
 
 echo "Showing details of the batch endpoint"
@@ -30,7 +32,7 @@ az ml batch-endpoint show --name $ENDPOINT_NAME
 
 echo "Showing details of the batch deployment"
 # <check_batch_deployment_detail>
-az ml batch-deployment show --name nonmlflowdp --endpoint-name $ENDPOINT_NAME
+az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NAME
 # </check_batch_deployment_detail>
 
 echo "Invoking batch endpoint with public URI (MNIST)"
@@ -67,9 +69,15 @@ fi
 echo "Invoke batch endpoint with specific output file name"
 # <start_batch_scoring_job_configure_output_settings>
 export OUTPUT_FILE_NAME=predictions_`echo $RANDOM`.csv
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://pipelinedata.blob.core.windows.net/sampledata/mnist --input-type uri_folder --output-path azureml://datastores/workspaceblobstore/paths/$ENDPOINT_NAME --set output_file_name=$OUTPUT_FILE_NAME --mini-batch-size 20 --instance-count 5 --query name -o tsv)
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://pipelinedata.blob.core.windows.net/sampledata/mnist --input-type uri_folder --output-path azureml://datastores/workspaceblobstore/paths/$ENDPOINT_NAME --set output_file_name=$OUTPUT_FILE_NAME --query name -o tsv)
 # </start_batch_scoring_job_configure_output_settings>
 
+echo "Invoke batch endpoint with specific overwrites"
+# <start_batch_scoring_job_overwrite>
+export OUTPUT_FILE_NAME=predictions_`echo $RANDOM`.csv
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://pipelinedata.blob.core.windows.net/sampledata/mnist --input-type uri_folder --mini-batch-size 20 --instance-count 5 --query name -o tsv)
+# </start_batch_scoring_job_overwrite>
+
 echo "Stream job detail"
 # <stream_job_logs_to_console>
 az ml job stream -n $JOB_NAME
@@ -93,17 +101,19 @@ fi
 
 echo "List all jobs under the batch deployment"
 # <list_all_jobs>
-az ml batch-deployment list-jobs --name nonmlflowdp --endpoint-name $ENDPOINT_NAME --query [].name
+az ml batch-deployment list-jobs --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NAME --query [].name
 # </list_all_jobs>
 
-echo "Create a new batch deployment (mlflow-nyctaxi), not setting it as default this time"
+echo "Create a new batch deployment (mnist-keras-dpl), not setting it as default this time"
 # <create_new_deployment_not_default>
-az ml batch-deployment create --name mlflowdp --endpoint-name $ENDPOINT_NAME --file endpoints/batch/mlflow-deployment.yml
+az ml batch-deployment create --file endpoints/batch/mnist-keras-deployment.yml --endpoint-name $ENDPOINT_NAME
 # </create_new_deployment_not_default>
 
 echo "Invoke batch endpoint with public data"
 # <test_new_deployment>
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --deployment-name mlflowdp --input https://pipelinedata.blob.core.windows.net/sampledata/nytaxi/taxi-tip-data.csv --input-type uri_file --query name -o tsv)
+DEPLOYMENT_NAME="mnist-keras-dpl"
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --deployment-name $DEPLOYMENT_NAME --input https://pipelinedata.blob.core.windows.net/sampledata/mnist --input-type uri_folder --query name -o tsv)
+# </test_new_deployment>
 
 echo "Show job detail"
 # <show_job_in_studio>
@@ -130,30 +140,20 @@ else
   exit 2
 fi
 # </check_job_status>
-# </test_new_deployment>
 
 echo "Update the batch deployment as default for the endpoint"
 # <update_default_deployment>
-az ml batch-endpoint update --name $ENDPOINT_NAME --set defaults.deployment_name=mlflowdp
+az ml batch-endpoint update --name $ENDPOINT_NAME --set defaults.deployment_name=$DEPLOYMENT_NAME
 # </update_default_deployment>
 
 echo "Verify default deployment. In this example, it should be mlflowdp."
 # <verify_default_deployment>
 az ml batch-endpoint show --name $ENDPOINT_NAME --query "{Name:name, Defaults:defaults}"
-
-DEFAULT_DEPL_NAME=$(az ml batch-endpoint show --name $ENDPOINT_NAME --query defaults.deployment_name)
-if [[ $DEFAULT_DEPL_NAME == \"mlflowdp\" ]]
-then
-  echo "mlflowdp is set as default successfully"
-else 
-  echo "default deployment set failed"
-  exit 1
-fi
 # </verify_default_deployment>
 
 echo "Invoke batch endpoint with the new default deployment with public URI"
 # <test_new_default_deployment>
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://pipelinedata.blob.core.windows.net/sampledata/nytaxi/taxi-tip-data.csv --input-type uri_file --query name -o tsv)
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://pipelinedata.blob.core.windows.net/sampledata/mnist --input-type uri_folder --query name -o tsv)
 # </test_new_default_deployment>
 
 echo "Stream job logs to console"
@@ -196,7 +196,7 @@ RESPONSE=$(curl --location --request POST "$SCORING_URI" \
   \"properties\": {
     \"dataset\": {
       \"dataInputType\": \"DataUrl\",
-      \"Path\": \"https://pipelinedata.blob.core.windows.net/sampledata/nytaxi/taxi-tip-data.csv\"
+      \"Path\": \"https://pipelinedata.blob.core.windows.net/sampledata/mnist\"
     }
   }
 }")

cli/endpoints/batch/batch-endpoint.yml renamed to cli/endpoints/batch/mnist-endpoint.yml

@@ -1,4 +1,4 @@
 $schema: https://azuremlschemas.azureedge.net/latest/batchEndpoint.schema.json
-name: mybatchedp
-description: my sample batch endpoint
+name: mnist-batch
+description: A batch endpoint for scoring images from the MNIST dataset.
 auth_mode: aad_token

cli/endpoints/batch/mnist-keras-deployment.yml

@@ -0,0 +1,19 @@
+$schema: https://azuremlschemas.azureedge.net/latest/batchDeployment.schema.json
+name: mnist-keras-dpl
+description: A deployment using Keras with TensorFlow to solve the MNIST classification dataset.
+endpoint_name: mnist-batch
+model: 
+  path: ./mnist-keras/model/
+code_configuration:
+  code: ./mnist-keras/code/
+  scoring_script: batch_driver.py
+environment:
+  conda_file: ./mnist-keras/environment/conda.yml
+  image: mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:latest
+compute: azureml:batch-cluster
+resources:
+  instance_count: 1
+max_concurrency_per_instance: 2
+mini_batch_size: 10
+output_action: append_row
+output_file_name: predictions.csv

cli/endpoints/batch/mnist-keras/code/batch_driver.py

@@ -0,0 +1,37 @@
+import os
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+from os.path import basename
+from PIL import Image
+from tensorflow.keras.models import load_model
+
+
+def init():
+    global model
+
+    # AZUREML_MODEL_DIR is an environment variable created during deployment
+    model_path = os.path.join(os.environ["AZUREML_MODEL_DIR"], "model")
+
+    # load the model
+    model = load_model(model_path)
+
+
+def run(mini_batch):
+    results = []
+
+    for image in mini_batch:
+        data = Image.open(image)
+        data = np.array(data)
+        data_batch = tf.expand_dims(data, axis=0)
+
+        # perform inference
+        pred = model.predict(data_batch)
+
+        # Compute probabilities, classes and labels
+        pred_prob = tf.math.reduce_max(tf.math.softmax(pred, axis=-1)).numpy()
+        pred_class = tf.math.argmax(pred, axis=-1).numpy()
+
+        results.append([basename(image), pred_class[0], pred_prob])
+
+    return pd.DataFrame(results)

cli/endpoints/batch/mnist-keras/environment/conda.yml

@@ -0,0 +1,12 @@
+name: tensorflow-env
+channels:
+  - conda-forge
+dependencies:
+  - python=3.7
+  - pip
+  - pip:
+    - pandas
+    - tensorflow
+    - pillow
+    - azureml-core
+    - azureml-dataset-runtime[fuse]