[TT-1847] ocr3 keystone por test broken 2 - WIP, DO NOT REVIEW

.github/e2e-tests.yml

@@ -183,6 +183,22 @@ runner-test-matrix:
       E2E_TEST_CHAINLINK_VERSION: '{{ env.DEFAULT_CHAINLINK_PLUGINS_VERSION }}' # This is the chainlink version that has the plugins
       ENABLE_OTEL_TRACES: true
 
+  - id: capabilities/workflow_test.go:*
+    path: integration-tests/capabilities/workflow_test.go
+    test_env_type: docker
+    runs_on: ubuntu-latest
+    triggers:
+      - PR Workflow Engine E2E Core Tests
+      - Nightly E2E Tests
+    test_cmd: cd integration-tests/ && go test capabilities/workflow_test.go -timeout 30m -count=1 -test.parallel=1 -json
+    pyroscope_env: ci-smoke-capabilities-evm-simulated
+    test_env_vars:
+      ENABLE_OTEL_TRACES: true
+      IS_CI: "true"
+      CTF_CONFIGS: "environment-ci.toml"
+      # Anvil developer key, not a secret
+      PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+
   - id: chaos/ocr_chaos_test.go
     path: integration-tests/chaos/ocr_chaos_test.go
     test_env_type: k8s-remote-runner

.github/workflows/integration-tests.yml

@@ -108,6 +108,13 @@ jobs:
               - 'core/**/migrations/*.sql'
               - 'core/**/config/**/*.toml'
               - 'integration-tests/**/*.toml'
+            workflow_engine_changes:
+              - 'core/services/gateway/**'
+              - 'core/services/ocr3/**'
+              - 'core/services/ocrcommon/**'
+              - 'core/services/registrysyncer/**'
+              - 'core/services/workflows/**'
+              - 'core/services/standardcapabilities/**'
             ccip_changes:
               - '**/*ccip*'
               - '**/*ccip*/**'
@@ -119,6 +126,8 @@ jobs:
       github_ci_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.github_ci_changes }}
       core_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.core_changes }}
       ccip_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.ccip_changes }}
+      # set to 'true' for testing purposes
+      workflow_engine_changes: 'true'
 
   build-chainlink:
     environment: integration
@@ -165,6 +174,50 @@ jobs:
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           dep_evm_sha: ${{ inputs.evm-ref }}
 
+  run-core-e2e-workflow-engine-tests-for-pr:
+    name: Run Core Workflow Engine E2E Tests For PR
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    needs: [build-chainlink, changes]
+    if: github.event_name == 'pull_request' && ( needs.changes.outputs.workflow_engine_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
+    uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@6123809ec05787bfe0424fad755366ea915614c6
+    with:
+      workflow_name: Run Core Workflow Engine Tests For PR
+      chainlink_version: ${{ inputs.evm-ref || github.sha }}
+      chainlink_upgrade_version: ${{ github.sha }}
+      test_path: .github/e2e-tests.yml
+      test_trigger: PR Workflow Engine E2E Core Tests
+      upload_cl_node_coverage_artifact: true
+      upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
+      enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
+    secrets:
+      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+      PROD_AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
+      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
+      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
+      LOKI_URL: ${{ secrets.LOKI_URL }}
+      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
+      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
+      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+      MAIN_DNS_ZONE_PUBLIC_SDLC: ${{ secrets.MAIN_DNS_ZONE_PUBLIC_SDLC }}
+      AWS_K8S_CLUSTER_NAME_SDLC: ${{ secrets.AWS_K8S_CLUSTER_NAME_SDLC }}
+      OPTIONAL_GATI_AWS_ROLE_ARN: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
+      OPTIONAL_GATI_LAMBDA_URL: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
+
   run-core-e2e-tests-for-pr:
     name: Run Core E2E Tests For PR
     permissions:
@@ -174,7 +227,9 @@ jobs:
       id-token: write
       contents: read
     needs: [build-chainlink, changes]
-    if: github.event_name == 'pull_request' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
+    # disable for keystone smoke test testing
+    if: false
+    # if: github.event_name == 'pull_request' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
     uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@61acf908014e2fe42c04f2a507c79f97eb5f8c4d
     with:
       workflow_name: Run Core E2E Tests For PR
@@ -262,7 +317,9 @@ jobs:
       id-token: write
       contents: read
     needs: [build-chainlink, changes]
-    if: github.event_name == 'pull_request' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
+    # disable for keystone smoke test testing
+    if: false
+    # if: github.event_name == 'pull_request' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
     uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@61acf908014e2fe42c04f2a507c79f97eb5f8c4d
     with:
       workflow_name: Run CCIP E2E Tests For PR
@@ -343,28 +400,39 @@ jobs:
     if: always()
     name: ETH Smoke Tests
     runs-on: ubuntu-latest
-    needs: [run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
+    # needs: [run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
+    needs: [run-core-e2e-workflow-engine-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
     steps:
-      - name: Check Core test results
-        id: check_core_results
+      # - name: Check Core test results
+      #   id: check_core_results
+      #   run: |
+      #     results='${{ needs.run-core-e2e-tests-for-pr.outputs.test_results }}'
+      #     echo "Core test results:"
+      #     echo "$results" | jq .
+
+      #     node_migration_tests_failed=$(echo $results | jq '[.[] | select(.id == "integration-tests/migration/upgrade_version_test.go:*" ) | select(.result != "success")] | length > 0')
+      #     echo "node_migration_tests_failed=$node_migration_tests_failed" >> $GITHUB_OUTPUT
+
+      - name: Check Core Workflow Engine test results
+        id: check_core_workflow_engine_results
         run: |
-          results='${{ needs.run-core-e2e-tests-for-pr.outputs.test_results }}'
+          results='${{ needs.run-core-e2e-workflow-engine-tests-for-p.outputs.test_results }}'
           echo "Core test results:"
           echo "$results" | jq .
 
           node_migration_tests_failed=$(echo $results | jq '[.[] | select(.id == "integration-tests/migration/upgrade_version_test.go:*" ) | select(.result != "success")] | length > 0')
           echo "node_migration_tests_failed=$node_migration_tests_failed" >> $GITHUB_OUTPUT
 
-      - name: Check CCIP test results
-        id: check_ccip_results
-        run: |
-          if [[ '${{ needs.run-ccip-e2e-tests-for-pr.result }}' != 'skipped' ]]; then
-            results='${{ needs.run-ccip-e2e-tests-for-pr.outputs.test_results }}'
-            echo "CCIP test results:"
-            echo "$results" | jq .
-          else
-            echo "CCIP tests were skipped."
-          fi
+      # - name: Check CCIP test results
+      #   id: check_ccip_results
+      #   run: |
+      #     if [[ '${{ needs.run-ccip-e2e-tests-for-pr.result }}' != 'skipped' ]]; then
+      #       results='${{ needs.run-ccip-e2e-tests-for-pr.outputs.test_results }}'
+      #       echo "CCIP test results:"
+      #       echo "$results" | jq .
+      #     else
+      #       echo "CCIP tests were skipped."
+      #     fi
 
       - name: Send slack notification for failed migration tests
         if: steps.check_core_results.outputs.node_migration_tests_failed == 'true' && github.event_name != 'workflow_dispatch'
@@ -406,7 +474,8 @@ jobs:
 
   show-chainlink-node-coverage:
     name: Show Chainlink Node Go Coverage
-    if: always()
+    if: false
+    # if: always()
     needs: [run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
     runs-on: ubuntu-latest
     steps:
@@ -427,8 +496,10 @@ jobs:
 
   ## Solana Section
   get_solana_sha:
-    # We don't directly merge dependabot PRs, so let's not waste the resources
-    if: ${{ github.actor != 'dependabot[bot]' && inputs.run_solana != 'false' }}
+    # disable for now
+    if: false
+    # We don't directly merge dependabot PRs, so let's not waste the resources\
+    # if: ${{ github.actor != 'dependabot[bot]' && inputs.run_solana != 'false' }}
     name: Get Solana Sha From Go Mod
     environment: Integration
     runs-on: ubuntu-latest