Update E2E tests reusable workflow and test workflow for automation (#…

…13896) * Update job names in E2E reusable workflow * Fix * Update run-automation-ondemand-e2e-tests.yml * Fix logs for k8s runner * Upload test log on failure * Fix * Do not detach runner for automation reorg and chaos tests * Fix "Run Migration Tests" step
smartcontractkit · Jul 19, 2024 · 0b2826e · 0b2826e
1 parent 6adb827
commit 0b2826e
Show file tree

Hide file tree

Showing 4 changed files with 124 additions and 101 deletions.
diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
@@ -46,15 +46,15 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 -json
     test_inputs:
       test_suite: soak
 
   - id: integration-tests/soak/ocr_test.go:^TestOCRv2Soak$
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -64,7 +64,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -74,7 +74,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -84,7 +84,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -94,7 +94,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -104,7 +104,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GasSpike$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GasSpike$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -114,7 +114,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_ChangeBlockGasLimit$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_ChangeBlockGasLimit$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -124,7 +124,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForAllCLNodes$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForAllCLNodes$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -134,7 +134,7 @@ runner-test-matrix:
     path: integration-tests/soak/ocr_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
-    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForHalfCLNodes$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForHalfCLNodes$ -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -550,17 +550,17 @@ runner-test-matrix:
     test_inputs:
       test_suite: reorg
     workflows:
-      # - Run Automation Product Nightly E2E Tests
-    test_cmd: cd integration-tests/reorg && go test -v -test.run ^TestAutomationReorg$ -test.parallel=5 -timeout 60m -count=1
+      - Run Automation On Demand Tests (TEST WORKFLOW)
+    test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg$ -test.parallel=7 -timeout 60m -count=1 -json
     pyroscope_env: ci-automation-on-demand-reorg
 
   - id: integration-tests/chaos/automation_chaos_test.go
     path: integration-tests/chaos/automation_chaos_test.go
     test_env_type: k8s-remote-runner
     runs_on: ubuntu-latest
     workflows:
-      # - Run Automation Product Nightly E2E Tests
-    test_cmd: cd integration-tests/chaos && go test -v -test.run ^TestAutomationChaos$ -test.parallel=15 -timeout 60m -count=1
+      - Run Automation On Demand Tests (TEST WORKFLOW)
+    test_cmd: cd integration-tests/chaos && DETACH_RUNNER=false go test -v -test.run ^TestAutomationChaos$ -test.parallel=15 -timeout 60m -count=1 -json
     pyroscope_env: ci-automation-on-demand-chaos
     test_inputs:
       test_suite: chaos
@@ -572,7 +572,7 @@ runner-test-matrix:
     runs_on: ubuntu-latest
     # workflows:
       # - Run Nightly E2E Tests
-    test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json
     pyroscope_env: ci-benchmark-automation-nightly
     test_inputs:
       test_suite: benchmark
@@ -585,7 +585,7 @@ runner-test-matrix:
     path: integration-tests/smoke/vrfv2_test.go
     runs_on: ubuntu22.04-8cores-32GB
     test_env_type: docker
-    test_cmd: cd integration-tests/smoke && go test -v -test.run TestVRFv2Basic -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests/smoke && go test -v -test.run TestVRFv2Basic -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     workflows:
@@ -595,7 +595,7 @@ runner-test-matrix:
     path: integration-tests/load/vrfv2plus/vrfv2plus_test.go
     runs_on: ubuntu22.04-8cores-32GB
     test_env_type: docker
-    test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusPerformance$ -test.parallel=1 -timeout 24h -count=1
+    test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusPerformance$ -test.parallel=1 -timeout 24h -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -607,7 +607,7 @@ runner-test-matrix:
     path: integration-tests/load/vrfv2plus/vrfv2plus_test.go
     runs_on: ubuntu22.04-8cores-32GB
     test_env_type: docker
-    test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1
+    test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -619,7 +619,7 @@ runner-test-matrix:
     path: integration-tests/load/vrfv2/vrfv2_test.go
     runs_on: ubuntu22.04-8cores-32GB
     test_env_type: docker
-    test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2Performance$ -test.parallel=1 -timeout 24h -count=1
+    test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2Performance$ -test.parallel=1 -timeout 24h -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -631,7 +631,7 @@ runner-test-matrix:
     path: integration-tests/load/vrfv2/vrfv2_test.go
     runs_on: ubuntu22.04-8cores-32GB
     test_env_type: docker
-    test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1
+    test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     test_inputs:
@@ -643,7 +643,7 @@ runner-test-matrix:
     path: integration-tests/smoke/vrfv2plus_test.go
     runs_on: ubuntu22.04-8cores-32GB
     test_env_type: docker
-    test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Link_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1
+    test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Link_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 -json
     test_config_override_required: true
     test_secrets_required: true
     workflows:

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -978,10 +978,6 @@ jobs:
           DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
           DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
           DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
-          DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
-          DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
-          DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }}
 
       - name: Upload Coverage Data
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3