TT-1956 Add support for e2e docker tests in Flakeguard #16376 (#16380)

* Bump Flakeguard * wip * update test * update test cmd * bump * bump * bump * bump * fix * bump * fix * bump * bump * bump * bump * fail test * bump * Run nightly workflow with flakeguard * bump * bump * Add extraArgs input to E2E tests workflow for customizable arguments * bump * bump * bump * bump * Add extraArgs input to nightly E2E tests workflow for customizable arguments * bump * bump * bump * bump * bump * revert tests * bump * bump * bump * bump * bump * bump * bump * trigger tests * Revert "trigger tests" This reverts commit dbb37c4.
smartcontractkit · Feb 17, 2025 · 4662445 · 4662445
1 parent 016bd81
commit 4662445
Show file tree

Hide file tree

Showing 16 changed files with 31 additions and 19 deletions.
diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
@@ -153,7 +153,7 @@ runner-test-matrix:
       - PR E2E Core Tests
       - Merge Queue E2E Core Tests
       - Nightly E2E Tests
-    test_cmd: cd integration-tests/ && go test smoke/forwarders_ocr2_test.go -timeout 30m -count=1 -test.parallel=2 -json
+    test_cmd: cd integration-tests && go test github.com/smartcontractkit/chainlink/integration-tests/smoke -v -run TestForwarderOCR2Basic -timeout 30m -count=1 -test.parallel=2 -json
     pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated
 
   - id: smoke/ocr2_test.go:*
@@ -742,7 +742,7 @@ runner-test-matrix:
       - PR E2E Core Tests
       - Merge Queue E2E Core Tests
       - Nightly E2E Tests
-    test_cmd: cd integration-tests/ && go test smoke/vrf_test.go -timeout 30m -count=1 -test.parallel=2 -json
+    test_cmd: cd integration-tests && go test github.com/smartcontractkit/chainlink/integration-tests/smoke -v -run "TestVRFBasic|TestVRFJobReplacement" -timeout 30m -count=1 -test.parallel=2 -json
     pyroscope_env: ci-smoke-vrf-evm-simulated
 
   - id: smoke/vrfv2_test.go:*

diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml
@@ -28,7 +28,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       test_path: .github/e2e-tests.yml
       test_ids: '${{ inputs.testType }}/automation_test.go:TestAutomationBenchmark'

diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml
@@ -23,7 +23,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       test_path: .github/e2e-tests.yml
       test_ids: 'load/automationv2_1/automationv2_1_test.go:TestLogTrigger'

diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml
@@ -10,7 +10,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       test_path: .github/e2e-tests.yml
       test_trigger: Automation Nightly Tests

diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml
@@ -168,7 +168,7 @@ jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       test_path: .github/e2e-tests.yml
       test_list: ${{ needs.set-tests-to-run.outputs.test_list }}

diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml
@@ -21,7 +21,7 @@ concurrency:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       test_path: .github/e2e-tests.yml
       chainlink_version: ${{ github.sha }}

diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
@@ -36,7 +36,7 @@ concurrency:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       test_path: .github/e2e-tests.yml
       test_trigger: E2E CCIP Load Tests

diff --git a/.github/workflows/integration-chaos-tests.yml b/.github/workflows/integration-chaos-tests.yml
@@ -15,7 +15,7 @@ on:
 jobs:
   run-e2e-tests-workflow-dispatch:
     name: Run E2E Tests (Workflow Dispatch)
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     if: github.event_name == 'workflow_dispatch'
     with:
       test_path: .github/e2e-tests.yml
@@ -48,7 +48,7 @@ jobs:
 
   run-e2e-tests-workflow:
     name: Run E2E Tests (Push and Sechedule)
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     if: github.event_name != 'workflow_dispatch'
     with:
       test_path: .github/e2e-tests.yml

diff --git a/.github/workflows/integration-in-memory-tests.yml b/.github/workflows/integration-in-memory-tests.yml
@@ -74,7 +74,7 @@ jobs:
       contents: read
     needs: changes
     if: github.event_name == 'pull_request' && ( needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
-    uses: smartcontractkit/.github/.github/workflows/run-integration-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-integration-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       workflow_name: Run CCIP Integration Tests For PR
       test_path: .github/integration-in-memory-tests.yml
@@ -98,7 +98,7 @@ jobs:
       contents: read
     needs: changes
     if: github.event_name == 'merge_group' && ( needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
-    uses: smartcontractkit/.github/.github/workflows/run-integration-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-integration-tests.yml@d7f8e299e891eafa428a37b5e856d929232c6e18
     with:
       workflow_name: Run CCIP Integration Tests For Merge Queue
       test_path: .github/integration-in-memory-tests.yml

diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml
@@ -43,7 +43,7 @@ on:
 jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       test_path: .github/e2e-tests.yml
       test_ids: ${{ inputs.testToRun}}

diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml
@@ -71,7 +71,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

diff --git a/.github/workflows/on-demand-vrfv2-smoke-tests.yml b/.github/workflows/on-demand-vrfv2-smoke-tests.yml
@@ -74,7 +74,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
@@ -71,7 +71,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

diff --git a/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml b/.github/workflows/on-demand-vrfv2plus-smoke-tests.yml
@@ -74,7 +74,7 @@ jobs:
   run-e2e-tests-workflow:
     name: Run E2E Tests
     needs: set-tests-to-run
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       custom_test_list_json: ${{ needs.set-tests-to-run.outputs.test_list }}
       chainlink_version: ${{ inputs.chainlink_version }}

diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml
@@ -16,18 +16,24 @@ on:
         description: 'Notify on Slack after tests'
         required: false
         type: string
+      extraArgs:
+        required: false
+        type: string
+        default: '{ "flakeguard_enable": "true", "flakeguard_run_count": "3" }'
+        description: 'JSON of extra arguments for the workflow.'     
 
 jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       chainlink_version: ${{ inputs.chainlink_version || 'develop' }}
       test_path: .github/e2e-tests.yml
       test_trigger: Nightly E2E Tests
       slack_notification_after_tests: ${{ inputs.slack_notification_after_tests || 'always' }}
       slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
       slack_notification_after_tests_name: Nightly E2E Tests
+      extraArgs: ${{ inputs.extraArgs }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml
@@ -29,20 +29,26 @@ on:
         default: 'Run E2E Tests'
         required: false
         type: string
+      extraArgs:
+        required: false
+        type: string
+        default: '{ "flakeguard_enable": "false", "flakeguard_run_count": "3" }'
+        description: 'JSON of extra arguments for the workflow.'        
 
 run-name: ${{ inputs.workflow_run_name }}
 
 jobs:
   call-run-e2e-tests-workflow:
     name: Run E2E Tests
-    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@f5baaf5e95d718546820cc5fecb42b6df410343d
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@59ca3330cde92af5974cfb5ca2504e67fcf19db3
     with:
       chainlink_version: ${{ github.event.inputs.chainlink_version }}
       test_path: .github/e2e-tests.yml
       test_ids: ${{ github.event.inputs.test_ids }}
       test_config_override_path: ${{ github.event.inputs.test_config_override_path }}
       with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }}
       test_secrets_override_key: ${{ github.event.inputs.test_secrets_override_key }}
+      extraArgs: ${{ github.event.inputs.extraArgs }}
     secrets:
       QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
       QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}