add custom tests

TykTechnologies · Oct 18, 2024 · a5fd8bc · a5fd8bc
1 parent 82a0dee
commit a5fd8bc
Show file tree

Hide file tree

Showing 5 changed files with 277 additions and 3 deletions.
diff --git a/.github/custom-tests/comparison/all/cluster.tf b/.github/custom-tests/comparison/all/cluster.tf
@@ -0,0 +1,19 @@
+project                   = "ce-team-zaid"
+cluster_location          = "us-west1-a"
+gke_version               = ""
+cluster_machine_type      = "c2-standard-4"
+service_machine_type      = ""
+upstream_machine_type     = "c2-standard-8"
+tests_machine_type        = "c2-standard-8"
+resources_machine_type    = ""
+dependencies_machine_type = "c2-standard-8"
+
+services_nodes_count     = 1
+upstream_nodes_count     = 1
+tests_nodes_count        = 1
+resource_nodes_count     = 1
+dependencies_nodes_count = 2
+
+tyk_enabled      = true
+kong_enabled     = true
+gravitee_enabled = true
diff --git a/.github/custom-tests/comparison/all/deployments.tf b/.github/custom-tests/comparison/all/deployments.tf
@@ -0,0 +1,41 @@
+kubernetes_config_context = "performance-testing"
+
+analytics_database_enabled    = true
+analytics_prometheus_enabled  = true
+auth_enabled                  = true
+auth_type                     = "authToken"
+quota_enabled                 = false
+quota_rate                    = 999999
+quota_per                     = 3600
+rate_limit_enabled            = true
+rate_limit_rate               = 999999
+rate_limit_per                = 60
+open_telemetry_enabled        = false
+open_telemetry_sampling_ratio = "0.5"
+
+hpa_enabled                 = false
+hpa_max_replica_count       = 10
+replica_count               = 4
+hpa_avg_cpu_util_percentage = 80
+external_traffic_policy     = "local"
+resources_requests_cpu      = "750m"
+resources_requests_memory   = "2048Mi"
+resources_limits_cpu        = "750m"
+resources_limits_memory     = "2048Mi"
+
+tyk_enabled         = true
+tyk_version         = "v5.6"
+tyk_deployment_type = "Deployment"
+tyk_go_gc           = 1600
+tyk_go_max_procs    = 8
+
+kong_enabled         = true
+kong_version         = "3.6"
+kong_deployment_type = "Deployment"
+
+gravitee_enabled         = true
+gravitee_version         = "4.4"
+gravitee_deployment_type = "Deployment"
+gravitee_nginx_enabled   = false
+
+grafana_service_type = "ClusterIP"
diff --git a/.github/custom-tests/comparison/all/tests.tf b/.github/custom-tests/comparison/all/tests.tf
@@ -0,0 +1,12 @@
+tyk_enabled      = true
+kong_enabled     = true
+gravitee_enabled = true
+
+tests_fortio_options = "size=20"
+tests_executor       = "constant-arrival-rate"
+tests_auth_key_count = 10
+tests_ramping_steps  = 10
+tests_duration       = 120
+tests_rate           = 5000
+tests_virtual_users  = 500
+tests_parallelism    = 4
diff --git a/.github/workflows/cluster.yml b/.github/workflows/cluster.yml
@@ -85,11 +85,11 @@ jobs:
           cd aks
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="cluster_location=${{ vars.AZURE_CLUSTER_LOCATION }}" \
             --var="cluster_machine_type=${{ vars.AZURE_CLUSTER_MACHINE_TYPE }}" \
             --var="upstream_machine_type=${{ vars.AZURE_UPSTREAM_MACHINE_TYPE }}" \
             --var="tests_machine_type=${{ vars.AZURE_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
             --auto-approve
 
       - name: EKS cluster ${{ inputs.action }}
@@ -98,11 +98,11 @@ jobs:
           cd eks
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="cluster_location=${{ vars.AWS_CLUSTER_LOCATION }}" \
             --var="cluster_machine_type=${{ vars.AWS_CLUSTER_MACHINE_TYPE }}" \
             --var="upstream_machine_type=${{ vars.AWS_UPSTREAM_MACHINE_TYPE }}" \
             --var="tests_machine_type=${{ vars.AWS_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
             --auto-approve
 
       - name: GKE cluster ${{ inputs.action }}
@@ -111,10 +111,10 @@ jobs:
           cd gke
           terraform init
           terraform ${{ inputs.action }} \
-            --var-file=main.tfvars \
             --var="project=${{ secrets.GCP_PROJECT }}" \
             --var="cluster_location=${{ vars.GCP_CLUSTER_LOCATION }}" \
             --var="cluster_machine_type=${{ vars.GCP_CLUSTER_MACHINE_TYPE }}" \
             --var="upstream_machine_type=${{ vars.GCP_UPSTREAM_MACHINE_TYPE }}" \
             --var="tests_machine_type=${{ vars.GCP_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
             --auto-approve
diff --git a/.github/workflows/custom_performance_test.yml b/.github/workflows/custom_performance_test.yml
@@ -0,0 +1,202 @@
+name: Custom Performance Test
+on:
+  workflow_dispatch:
+    inputs:
+      cloud:
+        description: 'Choose Cloud Provider'
+        required: true
+        type: choice
+        default: Azure
+        options:
+          - Azure
+          - AWS
+          - GCP
+      custom_cluster_descriptor:
+        description: 'Custom Cluster Descriptor'
+        required: true
+        type: string
+        default: '.github/custom-tests/comparison/all/cluster.tf'
+      custom_deployments_descriptor:
+        description: 'Custom Deployments Descriptor'
+        required: true
+        type: string
+        default: '.github/custom-tests/comparison/all/deployments.tf'
+      custom_tests_descriptor:
+        description: 'Custom Tests Descriptor'
+        required: true
+        type: string
+        default: '.github/custom-tests/comparison/all/tests.tf'
+
+env:
+  provider: ${{ inputs.cloud == 'Azure' && 'aks' || (inputs.cloud == 'AWS' && 'eks' || 'gke') }}
+
+concurrency:
+  group: ${{ inputs.cloud }}
+
+jobs:
+  performance_test:
+    name: "${{ inputs.cloud }} custom full performance test run"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Configure AKS credentials
+        if: ${{ inputs.cloud == 'Azure' }}
+        uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Configure AWS credentials
+        if: ${{ inputs.cloud == 'AWS' }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_CLUSTER_LOCATION }}
+
+      - name: Authenticate into gcloud
+        if: ${{ inputs.cloud == 'GCP' }}
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
+
+      - name: Install gcloud CLI
+        if: ${{ inputs.cloud == 'GCP' }}
+        uses: google-github-actions/[email protected]
+
+      - name: Install gcloud k8s auth component
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: gcloud components install gke-gcloud-auth-plugin
+
+      - name: Install Terraform
+        uses: hashicorp/[email protected]
+        with:
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+          terraform_version: "1.8.2"
+
+      - name: Create Terraform Cloud descriptors
+        run: |
+          cp ${{ env.provider }}/terraform.cloud.tf.example ${{ env.provider }}/terraform.cloud.tf
+          cp .github/${{ env.provider }}/deployments.tf deployments/terraform.cloud.tf
+          cp .github/${{ env.provider }}/tests.tf tests/terraform.cloud.tf
+
+      - name: Copy Terraform descriptor
+        run: |
+          cp ${{ inputs.custom_cluster_descriptor }} ${{ env.provider }}/main.tfvars
+          cp ${{ inputs.custom_deployments_descriptor }} deployments/main.tfvars
+          cp ${{ inputs.custom_tests_descriptor }} tests/main.tfvars
+
+      - name: AKS cluster
+        if: ${{ inputs.cloud == 'Azure' }}
+        run: |
+          cd aks
+          terraform init
+          terraform apply \
+            --var="cluster_location=${{ vars.AZURE_CLUSTER_LOCATION }}" \
+            --var="cluster_machine_type=${{ vars.AZURE_CLUSTER_MACHINE_TYPE }}" \
+            --var="upstream_machine_type=${{ vars.AZURE_UPSTREAM_MACHINE_TYPE }}" \
+            --var="tests_machine_type=${{ vars.AZURE_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
+            --auto-approve
+
+      - name: EKS cluster
+        if: ${{ inputs.cloud == 'AWS' }}
+        run: |
+          cd eks
+          terraform init
+          terraform apply \
+            --var="cluster_location=${{ vars.AWS_CLUSTER_LOCATION }}" \
+            --var="cluster_machine_type=${{ vars.AWS_CLUSTER_MACHINE_TYPE }}" \
+            --var="upstream_machine_type=${{ vars.AWS_UPSTREAM_MACHINE_TYPE }}" \
+            --var="tests_machine_type=${{ vars.AWS_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
+            --auto-approve
+
+      - name: GKE cluster
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: |
+          cd gke
+          terraform init
+          terraform apply \
+            --var="project=${{ secrets.GCP_PROJECT }}" \
+            --var="cluster_location=${{ vars.GCP_CLUSTER_LOCATION }}" \
+            --var="cluster_machine_type=${{ vars.GCP_CLUSTER_MACHINE_TYPE }}" \
+            --var="upstream_machine_type=${{ vars.GCP_UPSTREAM_MACHINE_TYPE }}" \
+            --var="tests_machine_type=${{ vars.GCP_TEST_MACHINE_TYPE }}" \
+            --var-file=main.tfvars \
+            --auto-approve
+
+      - name: Connect to AKS cluster
+        if: ${{ inputs.cloud == 'Azure' }}
+        run: |
+          az aks get-credentials \
+            --resource-group "pt-${{ vars.AZURE_CLUSTER_LOCATION }}" \
+            --name "pt-${{ vars.AZURE_CLUSTER_LOCATION }}"
+
+          kubectl config rename-context $(kubectl config current-context) performance-testing
+
+      - name: Connect to EKS cluster
+        if: ${{ inputs.cloud == 'AWS' }}
+        run: |
+          aws eks --region "${{ vars.AWS_CLUSTER_LOCATION }}" update-kubeconfig --name "pt-${{ vars.AWS_CLUSTER_LOCATION }}"
+
+          kubectl config rename-context $(kubectl config current-context) performance-testing
+
+      - name: Connect to GKE cluster
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: |
+          gcloud container clusters get-credentials "pt-${{ vars.GCP_CLUSTER_LOCATION }}" \
+            --zone "${{ vars.GCP_CLUSTER_LOCATION }}" \
+            --project "${{ secrets.GCP_PROJECT }}"
+          
+          kubectl config rename-context $(kubectl config current-context) performance-testing
+
+      - name: Run Terraform action on deployments
+        run: |
+          cd deployments
+          terraform init
+          terraform apply \
+            --var="tyk_license=${{ secrets.DASH_LICENSE }}" \
+            --var-file=main.tfvars \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Run Tests
+        run: |
+          cd tests
+          terraform init
+          terraform apply \
+            --var-file=main.tfvars \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Test Grafana Snapshot
+        run: |
+          kubectl logs -n dependencies $(kubectl get pods -n dependencies --selector=app=snapshot-job -o jsonpath='{.items[-1].metadata.name}') --tail=1
+
+      - name: Destroy Tests
+        run: |
+          cd tests
+          terraform destroy \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Destroy testing resources
+        run: |
+          cd deployments
+          terraform destroy \
+            --var="kubernetes_config_context=performance-testing" \
+            --auto-approve
+
+      - name: Destroy AKS cluster
+        if: ${{ inputs.cloud == 'Azure' }}
+        run: cd aks && terraform destroy --auto-approve
+
+      - name: Destroy EKS cluster
+        if: ${{ inputs.cloud == 'AWS' }}
+        run: cd eks && terraform destroy --auto-approve
+
+      - name: Destroy GKE cluster
+        if: ${{ inputs.cloud == 'GCP' }}
+        run: cd gke && terraform destroy --auto-approve