squash

.github/workflows/auto_merge.yml

@@ -1,63 +1,232 @@
-name: Automatic PR Merger
-
+name: Merge & Deploy
 on:
-  push: {} # update PR when base branch is updated
-  status: {} # try to merge when other checks are completed
-  pull_request_review: # try to merge after review
-    types:
-      - submitted
-      - edited
-      - dismissed
-  pull_request: # try to merge if labels have changed (white/black list)
-    types:
-      - labeled
-      - unlabeled
+  push: 
+    branches:
+    - 'k8s' #TODO: main
+
+env:
+  # These values are viewable in the Sara K8s CI entry of 1password
+  # If updating secrets please also update the 1password entries
+  GCLOUD_ZONE_ID: ${{ secrets.GCLOUD_ZONE_ID }}
+  GCLOUD_CLUSTER_NAME: ${{ secrets.GCLOUD_CLUSTER_NAME }}
+  GCLOUD_PROJECT_ID: ${{ secrets.GCLOUD_PROJECT_ID }}
+  RASA_X_IMAGE_NAME: ${{ secrets.RASA_X_IMAGE_NAME }}
+  ACTION_SERVER_IMAGE_NAME: ${{ secrets.ACTION_SERVER_IMAGE_NAME }}
+  RASA_X_DOMAIN: ${{ secrets.RASA_X_DOMAIN }}
+  SARA_GKE_SERVICE_ACCOUNT_NAME: ${{ secrets.SARA_GKE_SERVICE_ACCOUNT_NAME }}
+  RASA_X_DATABASE_PASSWORD: ${{ secrets.RASA_X_DATABASE_PASSWORD }}
+  RASA_X_PASSWORD: ${{ secrets.RASA_X_PASSWORD }}
+  RABBITMQ_PASSWORD: ${{ secrets.RABBITMQ_PASSWORD }}
+  REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
+  RASA_TOKEN: ${{ secrets.RASA_TOKEN }}
+  RASA_X_TOKEN: ${{ secrets.RASA_X_TOKEN }}
+  JWT_SECRET: ${{ secrets.JWT_SECRET }}
+  PASSWORDSALT: ${{ secrets.PASSWORDSALT }}
+  STATIC_IP: ${{ secrets.STATIC_IP }}
+  INGRESS_CERTIFICATE: ${{ secrets.INGRESS_CERTIFICATE }}
+  STORAGE_BUCKET_URL: ${{ secrets.STORAGE_BUCKET_URL }}
+  ACTION_SERVER_SECRET_NAME: ${{ secrets.ACTION_SERVER_SECRET_NAME }}
+
+  NAMESPACE: "sara"
+  RASA_X_USERNAME: "ci_user"
+  RELEASE_NAME: "rasa-x"
+  ACTION_SERVER_TAG: "latest"
+  RASA_X_VERSION: "0.37.1"
+  RASA_VERSION: "2.3.4"
+
+  # Due to the issue with openssl library for Google Cloud SDK (gcloud)
+  # (https://github.com/GoogleCloudPlatform/github-actions/issues/128)
+  # we use 297.0.01 version
+  GCLOUD_VERSION: "297.0.1"
 
 jobs:
-  # that's all. two steps are needed - if PR is mergeable according to
-  # branch protection rules it will be merged automatically
-  # and latest model file is trained & uploaded to Rasa X
-  mergepal:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: rasahq/[email protected]
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-  train-upload-model:
-    name: Train and Upload Model to Rasa X
+  check_changed_files:
     runs-on: ubuntu-latest
+    outputs:
+      training: ${{ steps.check_training.outputs.training }}
+      actions: ${{ steps.check_actions.outputs.actions }}
     steps:
     - uses: actions/checkout@v1
     - id: files
       uses: jitterbit/get-changed-files@v1
-    - name: set_training
+    - id: check_training
       if: |
-          contains(  steps.files.outputs.all, 'data/' )
+        contains(  steps.files.outputs.all, 'data/' )
           || contains(  steps.files.outputs.all, 'config.yml' )
           || contains(  steps.files.outputs.all, 'domain.yml' )
-      run: echo "RUN_TRAINING=true" >> $GITHUB_ENV
+          || contains(  steps.files.outputs.all, 'requirements.txt' )
+      run: echo "::set-output name=training::true"
+    - id: check_actions
+      if: |
+        contains(  steps.files.outputs.all, 'actions/' )
+          || contains(  steps.files.outputs.all, 'Dockerfile' )
+      run: echo "::set-output name=actions::true"
+  build-push-action-server:
+    name: Build Action Server Docker Image
+    needs: 
+    - check_changed_files
+    runs-on: ubuntu-latest
+    if:  ${{needs.check_changed_files.outputs.actions}}
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@v2
+
+      - name: Authenticate into Google Cloud Platform
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '275.0.0'
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
+
+      - name: Configure Docker to use Google Cloud Platform ☁️
+        run: |
+          gcloud auth configure-docker
+
+      - name: Pull Latest Image
+        run: |
+          docker pull gcr.io/replicated-test/rasa-demo:latest || true
+
+      - name: Set Build ID from run ID and number
+        run: echo "BUILD_NUMBER=$GITHUB_RUN_NUMBER-$GITHUB_RUN_ID" >> $GITHUB_ENV
+
+      - name: Build Image
+        run: |
+          docker build -t gcr.io/replicated-test/rasa-demo:run$BUILD_NUMBER -t gcr.io/replicated-test/rasa-demo:latest --cache-from gcr.io/replicated-test/rasa-demo:latest .
+      
+      - name: Push PR Image to Google Cloud Container Registry
+        run: |
+          docker push gcr.io/replicated-test/rasa-demo
+
+  deploy_to_k8s_cluster:
+    name: Rasa Enterprise K8s deployment
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/[email protected]
+      with:
+        access_token: ${{ github.token }}
+    - name: Checkout git repository 🕝
+      uses: actions/checkout@v2
+
+    - name: Install Helm and helmfile ⛑
+      run: |
+        curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+        chmod 700 get_helm.sh
+        ./get_helm.sh
+
+        sudo curl -fsSL https://github.com/roboll/helmfile/releases/download/v0.130.0/helmfile_linux_amd64 --output /usr/local/bin/helmfile
+        sudo chmod +x /usr/local/bin/helmfile
+
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@e23988b2af9696c66e87d1efbc688d3a80c3be14
+      name: Authenticate with gcloud 🎫
+      with:
+        version: "${{ env.GCLOUD_VERSION }}"
+        service_account_email: ${{ env.SARA_GKE_SERVICE_ACCOUNT_NAME }}
+        service_account_key: ${{ secrets.SARA_GKE_SERVICE_ACCOUNT_KEY }}
+
+    - name: Authenticate docker and configure cluster credentials 🎫
+      run: |
+        # Set up docker to authenticate via gcloud command-line tool.
+        gcloud  --quiet auth configure-docker
+        gcloud container clusters get-credentials "$GCLOUD_CLUSTER_NAME" --project ${GCLOUD_PROJECT_ID} --zone "$GCLOUD_ZONE_ID"
+
+    - name: Prepare namespace
+      run: |
+        kubectl config set-context --current --namespace="${NAMESPACE}"
+
+    - name: Deploy Rasa X chart ☸️
+      run: |
+        cd ${{ github.workspace }}/.github/deployments && 
+        helmfile repos &&
+        helmfile --environment development sync
+
+    - name: Wait for deployment to be ready ⏰
+      timeout-minutes: 15
+      run: |
+        kubectl wait \
+          --for=condition=available \
+          --timeout=600s \
+          -l "app.kubernetes.io/component=rasa-x" deployment
+
+        # Wait for DB migration to be done
+        until [[ $(curl -s "https://${{ env.RASA_X_DOMAIN }}/api/health" | tee /tmp/output_status.txt | jq -r .database_migration.status) -eq "completed" ]]
+        do
+          cat /tmp/output_status.txt || true
+          sleep 5
+        done
+        # Wait for deployment to be ready
+        until [[ $(curl -s https://${{ env.RASA_X_DOMAIN }}/api/health | tee /tmp/output_health.txt | jq -r .production.status) -eq 200 ]]
+        do
+          cat /tmp/output_health.txt || true
+          sleep 5
+        done
+
+  train-upload-model:
+    name: Train and Upload Model to Rasa X
+    runs-on: ubuntu-latest
+    needs: 
+    - check_changed_files
+    - deploy_to_k8s_cluster
+    if: ${{needs.check_changed_files.outputs.training}}
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/[email protected]
+      with:
+        access_token: ${{ github.token }}
+    - uses: actions/checkout@v1
     - name: Set up Python 3.7
-      if: env.RUN_TRAINING == 'true'
       uses: actions/setup-python@v1
       with:
         python-version: 3.7
     - name: Install dependencies
-      if: env.RUN_TRAINING == 'true'
       run: |
           python -m pip install --upgrade "pip<20"
           pip install -r requirements-dev.txt
     - name: Train Model
-      if: env.RUN_TRAINING == 'true'
       working-directory: ${{ github.workspace }}
       run: |
-        rasa train
-    - name: Upload model
-      if: env.RUN_TRAINING == 'true'
-      env:
-        RASA_X_API_TOKEN: ${{ secrets.RASA_X_API_TOKEN }}
+          rasa train
+    - name: Set model name from Rasa version
+      run: |
+          rasa_version=$(python -c "import rasa; print(rasa.__version__)")
+          model_path=`ls models/*.tar.gz | head -n 1`
+          model_timestamp=$(basename "$model_path" .tar.gz)
+          model_name="$model_timestamp"_rasa"$rasa_version"
+          renamed_model_path=models/"$model_name".tar.gz
+          mv $model_path $renamed_model_path
+          echo "MODEL_NAME=${model_name} >> $GITHUB_ENV
+          echo "MODEL_PATH=${renamed_model_path}" >> $GITHUB_ENV
+
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@e23988b2af9696c66e87d1efbc688d3a80c3be14
+      name: Authenticate with gcloud 🎫
+      with:
+        version: "${{ env.GCLOUD_VERSION }}"
+        service_account_email: ${{ env.SARA_GKE_SERVICE_ACCOUNT_NAME }}
+        service_account_key: ${{ secrets.SARA_GKE_SERVICE_ACCOUNT_KEY }}
+
+    - name: Upload model to storage bucket
+      run: |
+          gsutil cp "${MODEL_PATH}" ${STORAGE_BUCKET_URL}/rasa_demo_models
+
+    - name: Upload model to Rasa X
       working-directory: ${{ github.workspace }}
       run: |
-        model_path=`ls models/*.tar.gz | head -n 1`
-        curl -k -F "model=@${model_path}" "https://website-demo.rasa.com/api/projects/default/models?api_token=${RASA_X_API_TOKEN}"
+           auth_token=$(curl --request POST \
+              --url https://${RASA_X_DOMAIN}/api/auth \
+              --header 'Content-Type: application/json' \
+              --data "{
+              \"username\": \"${RASA_X_USERNAME}\",
+              \"password\": \"${RASA_X_PASSWORD}\"
+            }" | jq -r '.access_token')
+
+            echo "AUTH_TOKEN=${auth_token}" >> $GITHUB_ENV
 
+            curl -k -F "model=@${MODEL_PATH}" \
+              --url https://$RASA_X_DOMAIN/api/projects/default/models \
+              --header "Authorization: Bearer ${auth_token}"
+
+    - name: Tag model as production
+      working-directory: ${{ github.workspace }}
+      run: |
+            curl -X PUT  \
+            --url https://$RASA_X_DOMAIN/api/projects/default/models/$MODEL_NAME/tags/production \
+            --header "Authorization: Bearer ${AUTH_TOKEN}"

.github/workflows/build_and_deploy.yml

@@ -151,12 +151,3 @@ jobs:
       if: env.ACTIONS_CHANGED == 'true'
       run: |
         docker build --cache-from gcr.io/replicated-test/rasa-demo:latest .
-
-  mergepal-merge: # run merge pal in the end
-    runs-on: ubuntu-latest
-    needs: [lint-testing, type-testing, training-testing, build-images]
-    steps:
-      - uses: actions/checkout@v1
-      - uses: rasahq/[email protected]
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -3,4 +3,3 @@ repos:
     rev: stable
     hooks:
     - id: black
-      language_version: python3.6

Dockerfile

@@ -1,5 +1,5 @@
 # Pull SDK image as base image
-FROM rasa/rasa-sdk:2.0.0
+FROM rasa/rasa-sdk:2.3.0
 
 # Use subdirectory as working directory
 WORKDIR /app

actions/actions.py

@@ -22,6 +22,7 @@
 from actions.api.discourse import DiscourseAPI
 from actions.api.gdrive_service import GDriveService
 from actions.api.mailchimp import MailChimpAPI
+from actions.api.rasaxapi import RasaXAPI
 
 logger = logging.getLogger(__name__)
 
@@ -774,13 +775,6 @@ def run(self, dispatcher, tracker, domain):
         return []
 
 
-def tag_convo(tracker: Tracker, label: Text) -> None:
-    """Tag a conversation in Rasa X with a given label"""
-    endpoint = f"http://{config.rasa_x_host}/api/conversations/{tracker.sender_id}/tags"
-    requests.post(url=endpoint, data=label)
-    return
-
-
 class ActionTagFeedback(Action):
     """Tag a conversation in Rasa X as positive or negative feedback """
 
@@ -803,7 +797,7 @@ def run(
         else:
             return []
 
-        tag_convo(tracker, label)
+        RasaXAPI.tag_convo(tracker, label)
 
         return []
 
@@ -829,6 +823,6 @@ def run(
         else:
             return []
 
-        tag_convo(tracker, label)
+        RasaXAPI.tag_convo(tracker, label)
 
         return []

actions/config.py

@@ -1,45 +1,21 @@
 import os
 
-policy_model_dir = os.environ.get("POLICY_MODEL_DIR", "models/dialogue/")
-
-rasa_nlu_config = os.environ.get("RASA_NLU_CONFIG", "nlu_config.yml")
-
-account_sid = os.environ.get("ACCOUNT_SID", "")
-
-auth_token = os.environ.get("AUTH_TOKEN", "")
-
-twilio_number = os.environ.get("TWILIO_NUMBER", "")
-
-platform_api = os.environ.get("RASA_API_ENDPOINT_URL", "")
-
-self_port = int(os.environ.get("SELF_PORT", "5001"))
-
-core_model_dir = os.environ.get("CORE_MODEL_DIR", "models/dialogue/")
-
-remote_core_endpoint = os.environ.get("RASA_REMOTE_CORE_ENDPOINT_URL", "")
-
-rasa_core_token = os.environ.get("RASA_CORE_TOKEN", "")
-
 mailchimp_api_key = os.environ.get("MAILCHIMP_API_KEY", "")
 
 mailchimp_list = os.environ.get("MAILCHIMP_LIST", "")
 
 gdrive_credentials = os.environ.get("GDRIVE_CREDENTIALS", "")
 
-access_token = os.environ.get("TELEGRAM_TOKEN", "")
-
-verify = os.environ.get("TELEGRAM_VERIFY", "rasas_bot")
-
-webhook_url = os.environ.get("WEBHOOK_URL", "https://website-demo.rasa.com/webhook")
-
-rasa_platform_token = os.environ.get("RASA_PLATFORM_TOKEN", "")
-
-rasa_nlg_endpoint = os.environ.get("RASA_NLG_ENDPOINT_URL", "")
-
 algolia_app_id = os.environ.get("ALGOLIA_APP_ID", "")
 
 algolia_search_key = os.environ.get("ALGOLIA_SEARCH_KEY", "")
 
 algolia_docs_index = os.environ.get("ALGOLIA_DOCS_INDEX", "")
 
 rasa_x_host = os.environ.get("RASA_X_HOST", "rasa-x:5002")
+
+rasa_x_password = os.environ.get("RASA_X_PASSWORD", "")
+
+rasa_x_username = os.environ.get("RASA_X_USERNAME", "")
+
+rasa_x_host_schema = os.environ.get("RASA_X_HOST_SCHEMA", "http")

endpoints.yml

@@ -1,2 +1,3 @@
 action_endpoint:
   url: http://localhost:5055/webhook
+