diff --git a/.github/actions/setup-create-base64-config/action.yml b/.github/actions/setup-create-base64-config/action.yml deleted file mode 100644 index a5e7c8f0b4d..00000000000 --- a/.github/actions/setup-create-base64-config/action.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Create Base64 Config -description: A composite action that creates a base64-encoded config to be used by integration tests - -inputs: - runId: - description: The run id - testLogCollect: - description: Whether to always collect logs, even for passing tests - default: "false" - selectedNetworks: - description: The networks to run tests against - chainlinkPostgresVersion: - description: The postgres version to use with the chainlink node - default: "15.6" - chainlinkVersion: - description: The git commit sha to use for the image tag - logstreamLogTargets: - description: Where to send logs (e.g. file, loki) - ethExecutionClient: - description: Ethereum execution client to use (geth, besu, nethermind or erigon) - customEthClientDockerImage: - description: custom docker image to use for eth client (e.g. hyperledger/besu:21.10.0) - -runs: - using: composite - steps: - - name: Prepare Base64 TOML override - shell: bash - id: base64-config-override - env: - RUN_ID: ${{ inputs.runId }} - TEST_LOG_COLLECT: ${{ inputs.testLogCollect }} - SELECTED_NETWORKS: ${{ inputs.selectedNetworks }} - CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }} - CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }} - LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }} - ETH_EXECUTION_CLIENT: ${{ inputs.ethExecutionClient }} - CUSTOM_ETH_CLIENT_DOCKER_IMAGE: ${{ inputs.customEthClientDockerImage }} - run: | - function convert_to_toml_array() { - local IFS=',' - local input_array=($1) - local toml_array_format="[" - - for element in "${input_array[@]}"; do - toml_array_format+="\"$element\"," - done - - toml_array_format="${toml_array_format%,}]" - echo "$toml_array_format" - } - - selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") - log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") - - if [ -n "$TEST_LOG_COLLECT" ]; then - test_log_collect=true - else - test_log_collect=false - fi - - custom_images="" - ethereum_version="" - - if [ -n "$CUSTOM_ETH_CLIENT_DOCKER_IMAGE" ]; then - ethereum_version="ethereum_version=\"\"" - custom_images+="[PrivateEthereumNetwork.CustomDockerImages]" - custom_images+=$'\n'"execution_layer=\"$CUSTOM_ETH_CLIENT_DOCKER_IMAGE\"" - fi - - if [ -n "$ETH_EXECUTION_CLIENT" ]; then - execution_layer="$ETH_EXECUTION_CLIENT" - else - execution_layer="geth" - fi - - grafana_bearer_token="" - if [ -n "$GRAFANA_BEARER_TOKEN" ]; then - grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\"" - fi - - cat << EOF > config.toml - [Network] - selected_networks=$selected_networks - - [ChainlinkImage] - version="$CHAINLINK_VERSION" - postgres_version="$CHAINLINK_POSTGRES_VERSION" - - [Logging] - test_log_collect=$test_log_collect - run_id="$RUN_ID" - - [Logging.LogStream] - log_targets=$log_targets - - [PrivateEthereumNetwork] - execution_layer="$execution_layer" - $ethereum_version - - $custom_images - EOF - - BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) - echo ::add-mask::$BASE64_CONFIG_OVERRIDE - echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV diff --git a/.github/actions/setup-merge-base64-config/action.yml b/.github/actions/setup-merge-base64-config/action.yml index b734873c1f6..79dc8758315 100644 --- a/.github/actions/setup-merge-base64-config/action.yml +++ b/.github/actions/setup-merge-base64-config/action.yml @@ -24,22 +24,20 @@ runs: echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV decoded_toml=$(echo $BASE64_CONFIG_OVERRIDE | base64 -d) - CHAINLINK_IMAGE=$(echo "$decoded_toml" | dasel -r toml 'ChainlinkImage.image') + CHAINLINK_IMAGE=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.image' 2>/dev/null || echo ''; }) echo ::add-mask::$CHAINLINK_IMAGE - CHAINLINK_VERSION=$(echo "$decoded_toml" | dasel -r toml 'ChainlinkImage.version') + CHAINLINK_VERSION=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.version' 2>/dev/null || echo ''; }) NETWORKS=$(echo "$decoded_toml" | awk -F'=' '/^[[:space:]]*selected_networks[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' 2>/dev/null) if [ -n "$CHAINLINK_IMAGE" ]; then echo "CHAINLINK_IMAGE=$CHAINLINK_IMAGE" >> $GITHUB_ENV else - echo "No Chainlink Image found in base64-ed config. Exiting" - exit 1 + echo "No Chainlink Image found in base64-ed config" fi if [ -n "$CHAINLINK_VERSION" ]; then echo "CHAINLINK_VERSION=$CHAINLINK_VERSION" >> $GITHUB_ENV else - echo "No Chainlink Version found in base64-ed config. Exiting" - exit 1 + echo "No Chainlink Version found in base64-ed config" fi if [ -n "$NETWORKS" ]; then echo "NETWORKS=$NETWORKS" >> $GITHUB_ENV diff --git a/.github/actions/setup-parse-base64-config/action.yml b/.github/actions/setup-parse-base64-config/action.yml index 5e03c6de284..72e8982e6d0 100644 --- a/.github/actions/setup-parse-base64-config/action.yml +++ b/.github/actions/setup-parse-base64-config/action.yml @@ -20,23 +20,21 @@ runs: shell: bash run: | decoded_toml=$(echo $BASE64_CONFIG_OVERRIDE | base64 -d) - CHAINLINK_IMAGE=$(echo "$decoded_toml" | dasel -r toml 'ChainlinkImage.image') + CHAINLINK_IMAGE=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.image' 2>/dev/null || echo ''; }) echo ::add-mask::$CHAINLINK_IMAGE - CHAINLINK_VERSION=$(echo "$decoded_toml" | dasel -r toml 'ChainlinkImage.version') + CHAINLINK_VERSION=$(echo "$decoded_toml" | { dasel -r toml 'ChainlinkImage.version' 2>/dev/null || echo ''; }) NETWORKS=$(echo "$decoded_toml" | awk -F'=' '/^[[:space:]]*selected_networks[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' 2>/dev/null) ETH2_EL_CLIENT=$(echo "$decoded_toml" | awk -F'=' '/^[[:space:]]*execution_layer[[:space:]]*=/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' 2>/dev/null) if [ -n "$CHAINLINK_IMAGE" ]; then echo "CHAINLINK_IMAGE=$CHAINLINK_IMAGE" >> $GITHUB_ENV else - echo "No Chainlink Image found in base64-ed config. Exiting" - exit 1 + echo "No Chainlink Image found in base64-ed config" fi if [ -n "$CHAINLINK_VERSION" ]; then echo "CHAINLINK_VERSION=$CHAINLINK_VERSION" >> $GITHUB_ENV else echo "No Chainlink Version found in base64-ed config. Exiting" - exit 1 fi if [ -n "$NETWORKS" ]; then echo "NETWORKS=$NETWORKS" >> $GITHUB_ENV diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml new file mode 100644 index 00000000000..6aef81ec12f --- /dev/null +++ b/.github/e2e-tests.yml @@ -0,0 +1,824 @@ +# This file specifies the GitHub runner for each E2E test and is utilized by all E2E CI workflows. +# +# Each entry in this file includes the following: +# - The GitHub runner (runs_on field) that will execute tests. +# - The tests that will be run by the runner. +# - The workflows (e.g., Run PR E2E Tests, Run Nightly E2E Tests) that should trigger these tests. +# +runner-test-matrix: + + # START: OCR tests + + # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go + - id: integration-tests/smoke/ocr_test.go:* + path: integration-tests/smoke/ocr_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -test.parallel=2 -json + pyroscope_env: ci-smoke-ocr-evm-simulated + + # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner + - id: integration-tests/smoke/ocr2_test.go:^TestOCRv2Request$ + path: integration-tests/smoke/ocr2_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-ocr2-evm-simulated-nightly + + - id: integration-tests/smoke/ocr2_test.go:^TestOCRv2Basic$ + path: integration-tests/smoke/ocr2_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-ocr2-evm-simulated-nightly + + # Example of a configuration for running a single soak test in Kubernetes Remote Runner + - id: integration-tests/soak/ocr_test.go:^TestOCRv1Soak$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestOCRv2Soak$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestForwarderOCRv1Soak$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestForwarderOCRv2Soak$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_GasSpike$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GasSpike$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_ChangeBlockGasLimit$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_ChangeBlockGasLimit$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_RPCDownForAllCLNodes$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForAllCLNodes$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_RPCDownForHalfCLNodes$ + path: integration-tests/soak/ocr_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForHalfCLNodes$ -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: soak + + - id: integration-tests/smoke/forwarder_ocr_test.go:* + path: integration-tests/smoke/forwarder_ocr_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/forwarder_ocr_test.go -timeout 30m -count=1 -test.parallel=2 -json + pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated + + - id: integration-tests/smoke/forwarders_ocr2_test.go:* + path: integration-tests/smoke/forwarders_ocr2_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/forwarders_ocr2_test.go -timeout 30m -count=1 -test.parallel=2 -json + pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated + + - id: integration-tests/smoke/ocr2_test.go:* + path: integration-tests/smoke/ocr2_test.go + test_env_type: docker + runs_on: ubuntu22.04-16cores-64GB + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -timeout 30m -count=1 -test.parallel=6 -json + pyroscope_env: ci-smoke-ocr2-evm-simulated + + - id: integration-tests/smoke/ocr2_test.go:*-plugins + path: integration-tests/smoke/ocr2_test.go + test_env_type: docker + runs_on: ubuntu22.04-16cores-64GB + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -timeout 30m -count=1 -test.parallel=6 -json + pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated + test_inputs: + # chainlink_version: '{{ env.GITHUB_SHA_PLUGINS }}' # This is the chainlink version that has the plugins + chainlink_version: develop-plugins + + # END: OCR tests + + # START: Automation tests + + - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_0|TestAutomationBasic/registry_2_1_conditional|TestAutomationBasic/registry_2_1_logtrigger$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_0|TestAutomationBasic/registry_2_1_conditional|TestAutomationBasic/registry_2_1_logtrigger$" -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_1_with_mercury_v02|TestAutomationBasic/registry_2_1_with_mercury_v03|TestAutomationBasic/registry_2_1_with_logtrigger_and_mercury_v02$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_1_with_mercury_v02|TestAutomationBasic/registry_2_1_with_mercury_v03|TestAutomationBasic/registry_2_1_with_logtrigger_and_mercury_v02$" -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_2_conditional|TestAutomationBasic/registry_2_2_logtrigger|TestAutomationBasic/registry_2_2_with_mercury_v02$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_2_conditional|TestAutomationBasic/registry_2_2_logtrigger|TestAutomationBasic/registry_2_2_with_mercury_v02$" -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_2_with_mercury_v03|TestAutomationBasic/registry_2_2_with_logtrigger_and_mercury_v02$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_2_with_mercury_v03|TestAutomationBasic/registry_2_2_with_logtrigger_and_mercury_v02$" -test.parallel=2 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_3_conditional_native|TestAutomationBasic/registry_2_3_conditional_link$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_conditional_native|TestAutomationBasic/registry_2_3_conditional_link$" -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_3_logtrigger_native|TestAutomationBasic/registry_2_3_logtrigger_link$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_logtrigger_native|TestAutomationBasic/registry_2_3_logtrigger_link$" -test.parallel=2 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_3_with_mercury_v03_link|TestAutomationBasic/registry_2_3_with_logtrigger_and_mercury_v02_link$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_with_mercury_v03_link|TestAutomationBasic/registry_2_3_with_logtrigger_and_mercury_v02_link$" -test.parallel=2 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestSetUpkeepTriggerConfig$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestSetUpkeepTriggerConfig$ -test.parallel=2 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationAddFunds$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationAddFunds$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationPauseUnPause$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPauseUnPause$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationRegisterUpkeep$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationRegisterUpkeep$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationPauseRegistry$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPauseRegistry$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationKeeperNodesDown$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationKeeperNodesDown$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationPerformSimulation$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPerformSimulation$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestAutomationCheckPerformGasLimit$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationCheckPerformGasLimit$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestUpdateCheckData$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestUpdateCheckData$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/automation_test.go:^TestSetOffchainConfigWithMaxGasPrice$ + path: integration-tests/smoke/automation_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestSetOffchainConfigWithMaxGasPrice$ -test.parallel=2 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-automation-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperBasicSmoke$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperBasicSmoke$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperBlockCountPerTurn$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperBlockCountPerTurn$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperSimulation$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperSimulation$ -test.parallel=2 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperCheckPerformGasLimit$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperCheckPerformGasLimit$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperRegisterUpkeep$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperRegisterUpkeep$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperAddFunds$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperAddFunds$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperRemove$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperRemove$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperPauseRegistry$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperPauseRegistry$ -test.parallel=2 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperMigrateRegistry$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperMigrateRegistry$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperNodeDown$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperNodeDown$ -test.parallel=3 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperPauseUnPauseUpkeep$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperPauseUnPauseUpkeep$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperUpdateCheckData$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperUpdateCheckData$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/smoke/keeper_test.go:^TestKeeperJobReplacement$ + path: integration-tests/smoke/keeper_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperJobReplacement$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-keeper-evm-simulated + + - id: integration-tests/load/automationv2_1/automationv2_1_test.go:TestLogTrigger + path: integration-tests/load/automationv2_1/automationv2_1_test.go + runs_on: ubuntu-latest + test_env_type: k8s-remote-runner + test_cmd: cd integration-tests/load/automationv2_1 && go test -test.run TestLogTrigger -test.parallel=1 -timeout 60m -count=1 -json + remote_runner_memory: 4Gi + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_suite: automationv2_1 + workflows: + - Automation Load Test + pyroscope_env: automation-load-test + + - id: integration-tests/smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0 + path: integration-tests/smoke/automation_upgrade_test.go + test_env_type: docker + runs_on: ubuntu22.04-8cores-32GB + workflows: + - Run Automation Product Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json + test_inputs: + chainlink_image: public.ecr.aws/chainlink/chainlink + chainlink_version: latest + chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' + chainlink_upgrade_version: develop + pyroscope_env: ci-smoke-automation-upgrade-tests + + - id: integration-tests/smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1 + path: integration-tests/smoke/automation_upgrade_test.go + test_env_type: docker + runs_on: ubuntu22.04-8cores-32GB + workflows: + - Run Automation Product Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json + test_inputs: + chainlink_image: public.ecr.aws/chainlink/chainlink + chainlink_version: latest + chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' + chainlink_upgrade_version: develop + pyroscope_env: ci-smoke-automation-upgrade-tests + + - id: integration-tests/smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2 + path: integration-tests/smoke/automation_upgrade_test.go + test_env_type: docker + runs_on: ubuntu22.04-8cores-32GB + workflows: + - Run Automation Product Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json + test_inputs: + chainlink_image: public.ecr.aws/chainlink/chainlink + chainlink_version: latest + chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' + chainlink_upgrade_version: develop + pyroscope_env: ci-smoke-automation-upgrade-tests + + - id: integration-tests/reorg/automation_reorg_test.go + path: integration-tests/reorg/automation_reorg_test.go + runs_on: ubuntu-latest + test_env_type: k8s-remote-runner + test_inputs: + test_suite: reorg + workflows: + # - Run Automation Product Nightly E2E Tests + test_cmd: cd integration-tests/reorg && go test -v -test.run ^TestAutomationReorg$ -test.parallel=5 -timeout 60m -count=1 + pyroscope_env: ci-automation-on-demand-reorg + + - id: integration-tests/chaos/automation_chaos_test.go + path: integration-tests/chaos/automation_chaos_test.go + test_env_type: k8s-remote-runner + runs_on: ubuntu-latest + workflows: + # - Run Automation Product Nightly E2E Tests + test_cmd: cd integration-tests/chaos && go test -v -test.run ^TestAutomationChaos$ -test.parallel=15 -timeout 60m -count=1 + pyroscope_env: ci-automation-on-demand-chaos + test_inputs: + test_suite: chaos + + - id: integration-tests/benchmark/keeper_test.go:^TestAutomationBenchmark$ + path: integration-tests/benchmark/keeper_test.go + test_env_type: k8s-remote-runner + remote_runner_memory: 4Gi + runs_on: ubuntu-latest + # workflows: + # - Run Nightly E2E Tests + test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 + pyroscope_env: ci-benchmark-automation-nightly + test_inputs: + test_suite: benchmark + + # END: Automation tests + + # START: VRF tests + + - id: integration-tests/smoke/vrfv2_test.go:TestVRFv2Basic + path: integration-tests/smoke/vrfv2_test.go + runs_on: ubuntu22.04-8cores-32GB + test_env_type: docker + test_cmd: cd integration-tests/smoke && go test -v -test.run TestVRFv2Basic -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + workflows: + - On Demand VRFV2 Smoke Test (Ethereum clients) + + - id: integration-tests/load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusPerformance$Smoke + path: integration-tests/load/vrfv2plus/vrfv2plus_test.go + runs_on: ubuntu22.04-8cores-32GB + test_env_type: docker + test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusPerformance$ -test.parallel=1 -timeout 24h -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_type: Smoke + workflows: + - On Demand VRFV2 Plus Performance Test + + - id: integration-tests/load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusBHSPerformance$Smoke + path: integration-tests/load/vrfv2plus/vrfv2plus_test.go + runs_on: ubuntu22.04-8cores-32GB + test_env_type: docker + test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_type: Smoke + workflows: + - On Demand VRFV2 Plus Performance Test + + - id: integration-tests/load/vrfv2/vrfv2_test.go:^TestVRFV2Performance$Smoke + path: integration-tests/load/vrfv2/vrfv2_test.go + runs_on: ubuntu22.04-8cores-32GB + test_env_type: docker + test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2Performance$ -test.parallel=1 -timeout 24h -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_type: Smoke + workflows: + - On Demand VRFV2 Performance Test + + - id: integration-tests/load/vrfv2/vrfv2_test.go:^TestVRFV2PlusBHSPerformance$Smoke + path: integration-tests/load/vrfv2/vrfv2_test.go + runs_on: ubuntu22.04-8cores-32GB + test_env_type: docker + test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1 + test_config_override_required: true + test_secrets_required: true + test_inputs: + test_type: Smoke + workflows: + - On Demand VRFV2 Performance Test + + - id: integration-tests/smoke/vrfv2plus_test.go:^TestVRFv2Plus$/^Link_Billing$ + path: integration-tests/smoke/vrfv2plus_test.go + runs_on: ubuntu22.04-8cores-32GB + test_env_type: docker + test_cmd: cd integration-tests && go test -v -test.run ^TestVRFv2Plus$/^Link_Billing$ smoke/vrfv2plus_test.go -test.parallel=1 -timeout 30m -count=1 + test_config_override_required: true + test_secrets_required: true + workflows: + - On Demand VRFV2Plus Smoke Test (Ethereum clients) + + - id: integration-tests/smoke/vrf_test.go:* + path: integration-tests/smoke/vrf_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/vrf_test.go -timeout 30m -count=1 -test.parallel=2 -json + pyroscope_env: ci-smoke-vrf-evm-simulated + + - id: integration-tests/smoke/vrfv2_test.go:* + path: integration-tests/smoke/vrfv2_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/vrfv2_test.go -timeout 30m -count=1 -test.parallel=6 -json + pyroscope_env: ci-smoke-vrf2-evm-simulated + + - id: integration-tests/smoke/vrfv2plus_test.go:* + path: integration-tests/smoke/vrfv2plus_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/vrfv2plus_test.go -timeout 30m -count=1 -test.parallel=9 -json + pyroscope_env: ci-smoke-vrf2plus-evm-simulated + + # END: VRF tests + + # START: LogPoller tests + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerFewFiltersFixedDepth$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerFewFiltersFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerFewFiltersFinalityTag$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerFewFiltersFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosFixedDepth$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosFinalityTag$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosPostgresFinalityTag$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosPostgresFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosPostgresFixedDepth$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosPostgresFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerReplayFixedDepth$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerReplayFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerReplayFinalityTag$ + path: integration-tests/smoke/log_poller_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerReplayFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-log_poller-evm-simulated + + # END: LogPoller tests + + # START: Other tests + + - id: integration-tests/smoke/runlog_test.go:* + path: integration-tests/smoke/runlog_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/runlog_test.go -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-runlog-evm-simulated + + - id: integration-tests/smoke/cron_test.go:* + path: integration-tests/smoke/cron_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/cron_test.go -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-cron-evm-simulated + + - id: integration-tests/smoke/flux_test.go:* + path: integration-tests/smoke/flux_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/flux_test.go -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-flux-evm-simulated + + - id: integration-tests/smoke/reorg_above_finality_test.go:* + path: integration-tests/smoke/reorg_above_finality_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/reorg_above_finality_test.go -timeout 30m -count=1 -json + pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated + + - id: integration-tests/migration/upgrade_version_test.go:* + path: integration-tests/migration/upgrade_version_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - Run PR E2E Tests + - Run Nightly E2E Tests + test_cmd: cd integration-tests/migration && go test upgrade_version_test.go -timeout 30m -count=1 -test.parallel=2 -json + test_inputs: + chainlink_image: public.ecr.aws/chainlink/chainlink + chainlink_version: '{{ env.LATEST_CHAINLINK_RELEASE_VERSION }}' + chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' + chainlink_upgrade_version: develop + + # END: Other tests diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml index 6a589af49ae..c21171a83db 100644 --- a/.github/workflows/automation-benchmark-tests.yml +++ b/.github/workflows/automation-benchmark-tests.yml @@ -66,7 +66,7 @@ jobs: QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} suites: benchmark chaos reorg load - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 env: DETACH_RUNNER: true TEST_SUITE: benchmark diff --git a/.github/workflows/automation-load-tests.yml b/.github/workflows/automation-load-tests.yml index a237b5b7dd5..187ca3b5b66 100644 --- a/.github/workflows/automation-load-tests.yml +++ b/.github/workflows/automation-load-tests.yml @@ -11,6 +11,10 @@ on: required: true default: U02Q14G80TY type: string + test_secrets_override_key: + description: 'Key to run tests with custom test secrets' + required: false + type: string jobs: automation_load: @@ -82,7 +86,7 @@ jobs: QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} suites: benchmark chaos reorg load - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 env: RR_CPU: 4000m RR_MEM: 4Gi @@ -95,6 +99,7 @@ jobs: PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} with: test_command_to_run: cd integration-tests/load && go test -timeout 1h -v -run TestLogTrigger ./automationv2_1 -count=1 + test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} test_download_vendor_packages_command: make gomod cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml index 5925a8195f0..f018124624a 100644 --- a/.github/workflows/automation-nightly-tests.yml +++ b/.github/workflows/automation-nightly-tests.yml @@ -114,7 +114,7 @@ jobs: grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" grafanaBearerToken: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 env: TEST_SUITE: ${{ matrix.tests.suite }} with: diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml index f0046b341e3..83701ef7f9d 100644 --- a/.github/workflows/automation-ondemand-tests.yml +++ b/.github/workflows/automation-ondemand-tests.yml @@ -2,22 +2,22 @@ name: Automation On Demand Tests on: workflow_dispatch: inputs: - chainlinkVersion: - description: Chainlink image version to use + chainlinkVersionUpdate: + description: Chainlink image version to upgrade to required: false type: string - chainlinkImage: - description: Chainlink image repo to use (Leave empty to build from head/ref) + chainlinkImageUpdate: + description: Chainlink image repo to upgrade to (Leave empty to build from head/ref) options: - public.ecr.aws/chainlink/chainlink - QA_ECR type: choice - chainlinkVersionUpdate: + chainlinkVersion: description: Chainlink image version to use initially for upgrade test default: latest required: true type: string - chainlinkImageUpdate: + chainlinkImage: description: Chainlink image repo to use initially for upgrade test required: true options: @@ -155,7 +155,7 @@ jobs: - name: reorg id: reorg suite: reorg - nodes: 5 + nodes: 7 os: ubuntu-latest enabled: ${{ inputs.enableReorg }} pyroscope_env: ci-automation-on-demand-reorg @@ -204,23 +204,23 @@ jobs: run: | if [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT - echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT - echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT + else + echo "image=${{ inputs.chainlinkImage }}" >>$GITHUB_OUTPUT + fi + if [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then echo "upgrade_image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT - elif [[ "${{ inputs.chainlinkImage }}" == "public.ecr.aws/chainlink/chainlink" ]]; then - READ_CL_IMAGE="public.ecr.aws/chainlink/chainlink" - echo "image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT + else + echo "upgrade_image=${{ inputs.chainlinkImageUpdate }}" >>$GITHUB_OUTPUT + fi + if [[ -z "${{ inputs.chainlinkVersion }}" ]] && [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then + echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT + else echo "version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT - echo "upgrade_version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT - echo "upgrade_image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT fi - if [[ "${{ matrix.tests.type }}" == "upgrade" ]]; then - if [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then - echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT - elif [[ "${{ inputs.chainlinkImageUpdate }}" == "public.ecr.aws/chainlink/chainlink" ]]; then - echo "image=public.ecr.aws/chainlink/chainlink" >>$GITHUB_OUTPUT - fi - echo "version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT + if [[ -z "${{ inputs.chainlinkVersionUpdate }}" ]] && [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then + echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT + else + echo "upgrade_version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT fi - name: Setup GAP for Grafana uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 @@ -231,65 +231,21 @@ jobs: api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} # other inputs duplicate-authorization-header: "true" - - name: Prepare Base64 TOML config - env: - SELECTED_NETWORKS: ${{ matrix.tests.network }} - OLD_IMAGE: ${{ steps.determine-build.outputs.image }} - OLD_VERSION: ${{ steps.determine-build.outputs.version }} - UPGRADE_VERSION: ${{ steps.determine-build.outputs.upgrade_version }} - UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }} - RUN_ID: ${{ github.run_id }} - TEST_LOG_COLLECT: "true" - LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} - run: | - echo ::add-mask::$UPGRADE_IMAGE - echo ::add-mask::$OLD_IMAGE - - # load reusable functions - source ./.github/scripts/functions.sh - - selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") - - if [ -n "$TEST_LOG_COLLECT" ]; then - test_log_collect=true - else - test_log_collect=false - fi - - log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") - - cat << EOF > config.toml - [Network] - selected_networks=$selected_networks - - [ChainlinkImage] - image="$OLD_IMAGE" - version="$OLD_VERSION" - - [ChainlinkUpgradeImage] - image="$UPGRADE_IMAGE" - version="$UPGRADE_VERSION" - - [Logging] - test_log_collect=$test_log_collect - run_id="$RUN_ID" - - [Logging.LogStream] - log_targets=$log_targets - EOF - - BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) - echo ::add-mask::$BASE64_CONFIG_OVERRIDE - echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV + - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 if: ${{ matrix.tests.enabled == true }} - env: - TEST_SUITE: ${{ matrix.tests.suite }} with: test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_suite: ${{ matrix.tests.suite }} + test_config_chainlink_version: ${{ steps.determine-build.outputs.version }} + test_config_chainlink_upgrade_version: ${{ steps.determine-build.outputs.upgrade_version }} + test_config_selected_networks: ${{ matrix.tests.network }} + test_config_logging_run_id: ${{ github.run_id }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_config_test_log_collect: "true" cl_repo: ${{ steps.determine-build.outputs.image }} cl_image_tag: ${{ steps.determine-build.outputs.version }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -300,6 +256,8 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + DEFAULT_CHAINLINK_IMAGE: ${{ steps.determine-build.outputs.image }} + DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }} DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml index 95b6a81d6cf..311614033b0 100644 --- a/.github/workflows/client-compatibility-tests.yml +++ b/.github/workflows/client-compatibility-tests.yml @@ -550,16 +550,6 @@ jobs: api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} # other inputs duplicate-authorization-header: "true" - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - ethExecutionClient: ${{ matrix.evm_node.eth_implementation }} - customEthClientDockerImage: ${{ matrix.evm_node.docker_image }} - name: Prepare test log name run: | replace_special_chars() { @@ -588,10 +578,17 @@ jobs: # comment_on_pr: false # theme: 'dark' - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && touch .root_dir && go test -timeout 30m -count=1 -json ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_config_chainlink_version: ${{ needs.select-versions.outputs.chainlink_version }} + test_config_selected_networks: ${{ env.SELECTED_NETWORKS}} + test_config_logging_run_id: ${{ github.run_id }} + test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_config_private_ethereum_network_execution_layer: ${{ matrix.evm_node.eth_implementation || 'geth' }} + test_config_private_ethereum_network_custom_docker_image: ${{ matrix.evm_node.docker_image }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ needs.select-versions.outputs.chainlink_version }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} diff --git a/.github/workflows/integration-chaos-tests.yml b/.github/workflows/integration-chaos-tests.yml index f4160de2b0e..673614bf2c2 100644 --- a/.github/workflows/integration-chaos-tests.yml +++ b/.github/workflows/integration-chaos-tests.yml @@ -132,7 +132,7 @@ jobs: echo ::add-mask::$BASE64_CONFIG_OVERRIDE echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 with: test_command_to_run: cd integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index eb91a09d298..a54db66ba06 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -340,23 +340,19 @@ jobs: api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} # other inputs duplicate-authorization-header: "true" - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} + test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} + test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} + test_config_logging_run_id: ${{ github.run_id }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -458,23 +454,18 @@ jobs: api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} # other inputs duplicate-authorization-header: "true" - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} + test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} + test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} + test_config_logging_run_id: ${{ github.run_id }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -700,23 +691,18 @@ jobs: # other inputs duplicate-authorization-header: "true" - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} + test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} + test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} + test_config_logging_run_id: ${{ github.run_id }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} @@ -955,20 +941,13 @@ jobs: uses: ./.github/actions/setup-create-base64-upgrade-config with: selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkImage: ${{ env.CHAINLINK_IMAGE }} chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }} - upgradeImage: ${{ env.UPGRADE_IMAGE }} upgradeVersion: ${{ env.UPGRADE_VERSION }} runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - grafanaUrl: ${{ vars.GRAFANA_URL }} - grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - name: Run Migration Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -992,6 +971,7 @@ jobs: go_coverage_dest_dir: ${{ github.workspace }}/.covdata should_tidy: "false" DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} + DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ env.UPGRADE_IMAGE }} DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} @@ -1293,9 +1273,10 @@ jobs: echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke + test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }} publish_check_name: Solana Smoke Test Results diff --git a/.github/workflows/on-demand-keeper-smoke-tests.yml b/.github/workflows/on-demand-keeper-smoke-tests.yml index 6af4d1be0c3..75359c7501f 100644 --- a/.github/workflows/on-demand-keeper-smoke-tests.yml +++ b/.github/workflows/on-demand-keeper-smoke-tests.yml @@ -131,21 +131,18 @@ jobs: else echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" fi - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} ## Run this step when changes that require tests to be run are made - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} + test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} + test_config_logging_run_id: ${{ github.run_id }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ inputs.evm-ref || github.sha }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml index 8eb498f68cd..924b43829e5 100644 --- a/.github/workflows/on-demand-ocr-soak-test.yml +++ b/.github/workflows/on-demand-ocr-soak-test.yml @@ -27,7 +27,11 @@ on: required: true default: U01A2B2C3D4 type: string - + test_secrets_override_key: + description: 'Key to run tests with custom test secrets' + required: false + type: string + jobs: ocr_soak_test: name: OCR Soak Test @@ -89,7 +93,7 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 env: DETACH_RUNNER: true TEST_SUITE: soak @@ -101,6 +105,7 @@ jobs: with: test_command_to_run: cd ./integration-tests && go test -v -count=1 -run ^${{ github.event.inputs.testToRun }}$ ./soak test_download_vendor_packages_command: make gomod + test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml index f1de054cd82..5f24fa81c3d 100644 --- a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml +++ b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml @@ -6,7 +6,11 @@ on: description: base64-ed config required: true type: string - + test_secrets_override_key: + description: 'Key to run tests with custom test secrets' + required: false + type: string + jobs: vrfv2_smoke_test: name: VRFV2 Smoke Test with custom EL client client @@ -46,10 +50,11 @@ jobs: echo "### Execution client used" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -run TestVRFv2Basic ./smoke/vrfv2_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml index 6bcc378d9da..3d55c384580 100644 --- a/.github/workflows/on-demand-vrfv2-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2-performance-test.yml @@ -19,6 +19,11 @@ on: description: "Regex for tests to run" required: false default: "(TestVRFV2Performance)" + test_secrets_override_key: + description: 'Key to run tests with custom test secrets' + required: false + type: string + jobs: vrfv2_performance_test: name: VRFV2 Performance Test @@ -76,10 +81,11 @@ jobs: echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 with: test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2 test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} diff --git a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml index 6e8a232fa8a..58ecd39763d 100644 --- a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml +++ b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml @@ -5,7 +5,11 @@ on: base64Config: description: base64-ed config required: true - type: string + type: string + test_secrets_override_key: + description: 'Key to run tests with custom test secrets' + required: false + type: string jobs: vrfv2plus_smoke_test: @@ -46,10 +50,11 @@ jobs: echo "### Execution client used" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -run ^TestVRFv2Plus$/^Link_Billing$ ./smoke/vrfv2plus_test.go 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml index 9714cf13d3a..658736ab03f 100644 --- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml @@ -14,6 +14,11 @@ on: description: "Regex for tests to run" required: false default: "(TestVRFV2PlusPerformance)" + test_secrets_override_key: + description: 'Key to run tests with custom test secrets' + required: false + type: string + jobs: vrfv2plus_performance_test: name: VRFV2 Plus Performance Test @@ -71,10 +76,11 @@ jobs: echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27 with: test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2plus test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml new file mode 100644 index 00000000000..10a09df130e --- /dev/null +++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml @@ -0,0 +1,164 @@ +name: Run Automation On Demand Tests (TEST WORKFLOW) + +on: + workflow_dispatch: + inputs: + chainlinkImage: + description: Chainlink image repo to use to upgrade Chainlink Nodes from + options: + - QA_ECR + - public.ecr.aws/chainlink/chainlink + default: QA_ECR + type: choice + chainlinkVersion: + description: Chainlink image version to use to upgrade Chainlink Nodes from + required: false + default: develop + type: string + chainlinkImageUpdate: + description: Chainlink image repo to use to upgrade Chainlink Nodes to + required: true + options: + - QA_ECR + - public.ecr.aws/chainlink/chainlink + default: QA_ECR + type: choice + chainlinkVersionUpdate: + description: Chainlink image version to use to upgrade Chainlink Nodes to + default: develop + required: true + type: string + enableChaos: + description: Check to enable chaos tests + type: boolean + default: false + required: true + enableReorg: + description: Check to enable reorg tests + type: boolean + default: false + required: true + with_existing_remote_runner_version: + description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"' + required: false + type: string + +env: + CHAINLINK_VERSION: ${{ inputs.chainlinkVersion || github.sha }} + CHAINLINK_UPGRADE_VERSION: ${{ inputs.chainlinkVersionUpdate }} + +jobs: + # Set tests to run based on the workflow inputs + set-tests-to-run: + name: Set tests to run + runs-on: ubuntu-latest + outputs: + test_list: ${{ steps.set-tests.outputs.test_list }} + require_chainlink_image_versions_in_qa_ecr: ${{ steps.set-tests.outputs.require_chainlink_image_versions_in_qa_ecr }} + require_chainlink_plugin_versions_in_qa_ecr: ${{ steps.set-tests.outputs.require_chainlink_plugin_versions_in_qa_ecr }} + steps: + - name: Set tests to run + id: set-tests + run: | + if [[ "${{ github.event.inputs.chainlinkImage }}" == 'QA_ECR' ]]; then + # Instead of passing chainlink image as plain text input, {{ env.QA_CHAINLINK_IMAGE }} will be securely resolved in the E2E tests reusable workflow + CHAINLINK_IMAGE="'{{ env.QA_CHAINLINK_IMAGE }}'" + elif [[ "${{ github.event.inputs.chainlinkImage }}" == 'public.ecr.aws/chainlink/chainlink' ]]; then + CHAINLINK_IMAGE=public.ecr.aws/chainlink/chainlink + else + echo "Error: Not supported image source '${{ github.event.inputs.chainlinkImage }}'" + exit 1 + fi + + if [[ "${{ github.event.inputs.chainlinkImageUpdate }}" == 'QA_ECR' ]]; then + CHAINLINK_UPGRADE_IMAGE="'{{ env.QA_CHAINLINK_IMAGE }}'" + elif [[ "${{ github.event.inputs.chainlinkImageUpdate }}" == 'public.ecr.aws/chainlink/chainlink' ]]; then + CHAINLINK_UPGRADE_IMAGE=public.ecr.aws/chainlink/chainlink + else + echo "Error: Not supported image source '${{ github.event.inputs.chainlinkImageUpdate }}'" + exit 1 + fi + + CHAINLINK_VERSION="${{ env.CHAINLINK_VERSION }}" + CHAINLINK_UPGRADE_VERSION="${{ env.CHAINLINK_UPGRADE_VERSION }}" + + # Always run upgrade tests + cat > test_list.yaml <> test_list.yaml <> test_list.yaml <> $GITHUB_OUTPUT + + chainlink_image_versions="" + if [ "${{ github.event.inputs.chainlinkImage }}" = "QA_ECR" ]; then + chainlink_image_versions+="${{ env.CHAINLINK_VERSION }}," + fi + if [ "${{ github.event.inputs.chainlinkImageUpdate }}" = "QA_ECR" ]; then + chainlink_image_versions+="${{ env.CHAINLINK_UPGRADE_VERSION }}" + fi + echo "require_chainlink_image_versions_in_qa_ecr=$chainlink_image_versions" >> $GITHUB_OUTPUT + + call-run-e2e-tests-workflow: + name: Run E2E Tests + needs: set-tests-to-run + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + test_list: ${{ needs.set-tests-to-run.outputs.test_list }} + require_chainlink_image_versions_in_qa_ecr: ${{ needs.set-tests-to-run.outputs.require_chainlink_image_versions_in_qa_ecr }} + require_chainlink_plugin_versions_in_qa_ecr: ${{ needs.set-tests-to-run.outputs.require_chainlink_image_versions_in_qa_ecr }} + with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }} + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + diff --git a/.github/workflows/run-e2e-tests-reusable-workflow.yml b/.github/workflows/run-e2e-tests-reusable-workflow.yml new file mode 100644 index 00000000000..43896262f3f --- /dev/null +++ b/.github/workflows/run-e2e-tests-reusable-workflow.yml @@ -0,0 +1,711 @@ +# This is a reusable workflow that runs E2E tests for Chainlink. +# It is not meant to be run on its own. +name: Run E2E Tests +on: + workflow_call: + inputs: + chainlink_version: + description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha' + required: false + type: string + test_ids: + description: 'Run tests by test ids separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml' + required: false + type: string + test_list: + description: 'Base64 encoded list of tests (YML objects) to run. Example in run-automation-ondemand-e2e-tests.yml' + required: false + type: string + test_workflow: + description: 'Run tests by workflow name. Example: "Run Nightly E2E Tests"' + required: false + type: string + # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 + # test_config_override_base64: + # required: false + # description: The base64-encoded test config override + # type: string + enable_check_test_configurations: + description: 'Set to "true" to enable check-test-configurations job' + required: false + type: boolean + default: false + with_existing_remote_runner_version: + description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"' + required: false + type: string + require_chainlink_image_versions_in_qa_ecr: + description: 'Check Chainlink image versions to be present in QA ECR. If not, build and push the image to QA ECR. Takes comma separated list of Chainlink image versions. Example: "5733cdcda9a9fc6da6343798b119b2ae136146cd,0b7d2c497a508efa5a827714780d908b7b8eda19"' + required: false + type: string + require_chainlink_plugin_versions_in_qa_ecr: + description: 'Check Chainlink plugins versions to be present in QA ECR. If not, build and push the image to QA ECR. Takes comma separated list of Chainlink image versions. Example: "5733cdcda9a9fc6da6343798b119b2ae136146cd,0b7d2c497a508efa5a827714780d908b7b8eda19"' + required: false + type: string + slack_notification_after_tests: + description: 'Set to "true" to send a slack notification after the tests' + required: false + type: boolean + default: false + slack_notification_after_tests_channel_id: + description: 'Slack channel ID to send the notification to' + required: false + type: string + slack_notification_after_tests_name: + description: 'Name of the slack notification' + required: false + type: string + secrets: + TEST_SECRETS_OVERRIDE_BASE64: + required: false + QA_AWS_REGION: + required: true + QA_AWS_ROLE_TO_ASSUME: + required: true + QA_AWS_ACCOUNT_NUMBER: + required: true + QA_PYROSCOPE_INSTANCE: + required: true + QA_PYROSCOPE_KEY: + required: true + QA_KUBECONFIG: + required: true + GRAFANA_INTERNAL_TENANT_ID: + required: true + GRAFANA_INTERNAL_BASIC_AUTH: + required: true + GRAFANA_INTERNAL_HOST: + required: true + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: + required: true + GH_TOKEN: + required: true + AWS_REGION: + required: true + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: + required: true + AWS_API_GW_HOST_GRAFANA: + required: true + SLACK_BOT_TOKEN: + required: false + +env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + GITHUB_SHA_PLUGINS: ${{ github.sha }}-plugins + CHAINLINK_ENV_USER: ${{ github.actor }} + CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} + SELECTED_NETWORKS: SIMULATED + MOD_CACHE_VERSION: 1 + TEST_LOG_LEVEL: debug + +jobs: + validate-inputs: + name: Validate workflow inputs + runs-on: ubuntu-latest + outputs: + require_chainlink_image_versions_in_qa_ecr_matrix: ${{ steps.set-required-chainlink-image-versions-matrix.outputs.versions }} + require_chainlink_plugin_versions_in_qa_ecr_matrix: ${{ steps.set-required-chainlink-plugin-versions-matrix.outputs.versions }} + steps: + - name: Check input conditions + run: | + if [[ "${{ inputs.test_ids }}" != "" && "${{ inputs.test_workflow }}" != "" ]]; then + echo "::error::Error: Both 'test_ids' and 'test_workflow' are provided. Please specify only one." + exit 1 + fi + if [[ "${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}" != "" ]]; then + echo "Will run tests with custom test secrets" + fi + - name: Install jq + run: sudo apt-get install jq + - name: Create matrix for required Chainlink image versions + id: set-required-chainlink-image-versions-matrix + run: | + if [[ "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" != '' ]]; then + image_versions=$(echo "${{ inputs.require_chainlink_image_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end') + echo "versions=$image_versions" >> $GITHUB_OUTPUT + fi + - name: Create matrix for required Chainlink plugin versions + id: set-required-chainlink-plugin-versions-matrix + run: | + if [[ "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" != '' ]]; then + image_versions=$(echo "${{ inputs.require_chainlink_plugin_versions_in_qa_ecr }}" | jq -Rc 'split(",") | if . == [""] then [] else . end') + echo "versions=$image_versions" >> $GITHUB_OUTPUT + fi + + check-test-configurations: + name: Check test configurations + if: ${{ inputs.enable_check_test_configurations }} + needs: validate-inputs + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Setup Go + uses: ./.github/actions/setup-go + - name: Run Check Tests Command + run: | + cd integration-tests/ + if ! go run citool/main.go check-tests . ../.github/e2e-tests.yml; then + echo "::error::Some E2E test configurations have to be added to .github/e2e-tests.yml. This file defines Github CI configuration for each E2E test or set of E2E tests." && exit 1 + fi + + get_latest_chainlink_release_version: + name: Get latest Chainlink release version + runs-on: ubuntu-latest + environment: integration + outputs: + latest_chainlink_release_version: ${{ steps.get_latest_version.outputs.latest_version }} + steps: + - name: Get Latest Version + id: get_latest_version + run: | + untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GH_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name) + latest_version="${untrimmed_ver:1}" + echo "Latest Chainlink release version: $latest_version" + echo "latest_version=${latest_version}" >> "$GITHUB_OUTPUT" + # Check if latest_version is empty + if [ -z "$latest_version" ]; then + echo "Error: The latest_version is empty. The migration tests need a verison to run." + exit 1 + fi + + load-test-configurations: + name: Load test configurations + needs: [validate-inputs] + runs-on: ubuntu-latest + outputs: + run-docker-tests: ${{ steps.check-matrices.outputs.run-docker-tests }} + run-k8s-tests: ${{ steps.check-matrices.outputs.run-k8s-tests }} + docker-matrix: ${{ steps.set-docker-matrix.outputs.matrix }} + k8s-runner-matrix: ${{ steps.set-k8s-runner-matrix.outputs.matrix }} + steps: + - name: Checkout code + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Setup Go + uses: ./.github/actions/setup-go + - name: Install jq + run: sudo apt-get install jq + - name: Generate Docker Tests Matrix + id: set-docker-matrix + run: | + cd integration-tests/citool + MATRIX_JSON=$(go run main.go filter --file ${{ github.workspace }}/.github/e2e-tests.yml --test-env-type 'docker' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') + echo "Docker tests:" + echo "$MATRIX_JSON" | jq + echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + - name: Generate K8s Tests Matrix + id: set-k8s-runner-matrix + run: | + cd integration-tests/citool + MATRIX_JSON=$(go run main.go filter --file ${{ github.workspace }}/.github/e2e-tests.yml --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') + echo "K8s tests:" + echo "$MATRIX_JSON" | jq + echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + - name: Check Test Matrices + id: check-matrices + run: | + DOCKER_MATRIX_EMPTY=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []') + K8S_MATRIX_EMPTY=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq '.tests == null or .tests == []') + + # Check if jq commands succeeded + if [ $? -ne 0 ]; then + echo "JSON parse error occurred." + exit 1 + fi + + if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]]; then + echo "run-docker-tests=false" >> $GITHUB_OUTPUT + else + echo "run-docker-tests=true" >> $GITHUB_OUTPUT + fi + if [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then + echo "run-k8s-tests=false" >> $GITHUB_OUTPUT + else + echo "run-k8s-tests=true" >> $GITHUB_OUTPUT + fi + + # Check if both matrices are empty + if [[ "$DOCKER_MATRIX_EMPTY" == "true" ]] && [[ "$K8S_MATRIX_EMPTY" == "true" ]]; then + echo "No tests found for inputs: '${{ toJson(inputs) }}'. Both Docker and Kubernetes tests matrices are empty" + exit 1 + fi + shell: bash + + - name: Check if test config override is required for any test + shell: bash + run: | + # Check if the test config override is provided and skip the checks if it is non-empty + # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 + # if [ -n "${{ inputs.test_config_override_base64 }}" ]; then + # echo "Test config override provided. Skipping checks for tests requiring config override." + # exit 0 + # fi + + # Parse the JSON to check for test_config_override_required in Docker matrix + DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_config_override_required") and .test_config_override_required) | .id else empty end' -r) + # Parse the JSON to check for test_config_override_required in Kubernetes matrix + K8S_TESTS_REQUIRING_CONFIG_OVERRIDE=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_config_override_required") and .test_config_override_required) | .id else empty end' -r) + + # Determine if any tests require a configuration override + if [ ! -z "$DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE" ] || [ ! -z "$K8S_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then + echo "Tests in .github/e2e-tests.yml requiring test config override:" + if [ ! -z "$DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then + echo $DOCKER_TESTS_REQUIRING_CONFIG_OVERRIDE + fi + if [ ! -z "$K8S_TESTS_REQUIRING_CONFIG_OVERRIDE" ]; then + echo $K8S_TESTS_REQUIRING_CONFIG_OVERRIDE + fi + echo "::error::Error: Some of the tests require a test config override. Please see workflow logs and set 'test_config_override_base64' to run these tests." + exit 1 + else + echo "No tests require a configuration override. Proceeding without overrides." + fi + + - name: Check if test secrets are required for any test + shell: bash + run: | + # Check if the test secret key is provided and skip the checks if it is non-empty + if [ -n "${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }}" ]; then + echo "Test secret key provided. Skipping checks for tests requiring secrets." + exit 0 + fi + + # Parse the JSON to check for test_secrets_required in Docker matrix + DOCKER_TESTS_REQUIRING_SECRETS=$(echo '${{ steps.set-docker-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_secrets_required") and .test_secrets_required) | .id else empty end' -r) + # Parse the JSON to check for test_secrets_required in Kubernetes matrix + K8S_TESTS_REQUIRING_SECRETS=$(echo '${{ steps.set-k8s-runner-matrix.outputs.matrix }}' | jq 'if .tests then .tests[] | select(has("test_secrets_required") and .test_secrets_required) | .id else empty end' -r) + + # Determine if any tests require secrets + if [ ! -z "$DOCKER_TESTS_REQUIRING_SECRETS" ] || [ ! -z "$K8S_TESTS_REQUIRING_SECRETS" ]; then + echo "Tests in .github/e2e-tests.yml requiring custom test secrets:" + if [ ! -z "$DOCKER_TESTS_REQUIRING_SECRETS" ]; then + echo $DOCKER_TESTS_REQUIRING_SECRETS + fi + if [ ! -z "$K8S_TESTS_REQUIRING_SECRETS" ]; then + echo $K8S_TESTS_REQUIRING_SECRETS + fi + echo "::error::Error: Some of the tests require custom test secrets to run. Please see workflow logs and set 'test_secrets_override_key' to run these tests." + exit 1 + else + echo "No tests require secrets. Proceeding without additional secret setup." + fi + + # Build Chainlink images required for the tests + require-chainlink-image-versions-in-qa-ecr: + name: Build Chainlink image + needs: [validate-inputs, load-test-configurations] + if: ${{ needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix != '' }} + runs-on: ubuntu-latest + environment: integration + permissions: + id-token: write + contents: read + env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + strategy: + matrix: + version: ${{ fromJson(needs.validate-inputs.outputs.require_chainlink_image_versions_in_qa_ecr_matrix) }} + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + + - name: Build Chainlink image for ${{ matrix.version }} and push it to QA ECR + uses: ./.github/actions/build-chainlink-image + with: + dockerfile: core/chainlink.Dockerfile + git_commit_sha: ${{ matrix.version }} + tag_suffix: '' + check_image_exists: 'true' + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + + # Build Chainlink plugins required for the tests + require-chainlink-plugin-versions-in-qa-ecr: + name: Build Chainlink plugins + needs: [validate-inputs, load-test-configurations] + if: ${{ needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix != '' }} + runs-on: ubuntu-latest + environment: integration + permissions: + id-token: write + contents: read + env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + strategy: + matrix: + version: ${{ fromJson(needs.validate-inputs.outputs.require_chainlink_plugin_versions_in_qa_ecr_matrix) }} + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + + - name: Build Chainlink plugins image for ${{ matrix.version }} + uses: ./.github/actions/build-chainlink-image + with: + dockerfile: plugins/chainlink.Dockerfile + git_commit_sha: ${{ matrix.version }} + tag_suffix: '-plugins' + check_image_exists: 'true' + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + + # Run Docker tests + run-docker-tests: + name: Run docker tests (${{ matrix.tests.id }}) + needs: [load-test-configurations, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr, get_latest_chainlink_release_version] + # Run when none of the needed jobs fail or are cancelled (skipped or successful jobs are ok) + if: ${{ needs.load-test-configurations.outputs.run-docker-tests == 'true' && always() && !failure() && !cancelled() }} + runs-on: ${{ matrix.tests.runs_on }} + strategy: + fail-fast: false + matrix: ${{fromJson(needs.load-test-configurations.outputs.docker-matrix)}} + environment: integration + permissions: + actions: read + checks: write + pull-requests: write + id-token: write + contents: read + env: + LATEST_CHAINLINK_RELEASE_VERSION: ${{ needs.get_latest_chainlink_release_version.outputs.latest_chainlink_release_version }} + steps: + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 + with: + id: e2e_tests_${{ matrix.tests.id_sanitized }} + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Run E2E Tests / Run docker tests (${{ matrix.tests.id }}) + test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + continue-on-error: true + + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Install jq + run: sudo apt-get install -y jq + - name: Show test configuration + run: echo '${{ toJson(matrix.tests) }}' | jq . + - name: Setup Go + uses: ./.github/actions/setup-go + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 + id: setup-gap + with: + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + duplicate-authorization-header: "true" + + - name: Run tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 + env: + DETACH_RUNNER: true + with: + test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs + test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }} + # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 + # test_config_override_base64: ${{ inputs.test_config_override_base64 }} + test_config_chainlink_version: ${{ matrix.tests.test_inputs.chainlink_version || inputs.chainlink_version || github.sha }} + test_config_chainlink_upgrade_version: ${{ matrix.tests.test_inputs.chainlink_upgrade_version }} + test_config_chainlink_postgres_version: ${{ matrix.tests.test_inputs.chainlink_postgres_version }} + test_config_selected_networks: ${{ matrix.tests.test_inputs.selected_networks || env.SELECTED_NETWORKS}} + test_config_logging_run_id: ${{ github.run_id }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_type: ${{ matrix.tests.test_inputs.test_type }} + test_suite: ${{ matrix.tests.test_inputs.test_suite }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + artifacts_name: ${{ matrix.tests.id_sanitized }}-test-logs + artifacts_location: | + ./integration-tests/smoke/logs/ + /tmp/gotest.log + publish_check_name: ${{ matrix.tests.id_sanitized }} + token: ${{ secrets.GH_TOKEN }} + no_cache: true # Do not restore cache since go was already configured in the previous step + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: "" + should_tidy: "false" + go_coverage_src_dir: /var/tmp/go-coverage + go_coverage_dest_dir: ${{ github.workspace }}/.covdata + DEFAULT_CHAINLINK_IMAGE: ${{ matrix.tests.test_inputs.chainlink_image || env.CHAINLINK_IMAGE }} + DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ matrix.tests.test_inputs.chainlink_upgrade_image }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} + DEFAULT_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} + + # Run K8s tests using old remote runner + + prepare-remote-runner-test-image: + needs: [load-test-configurations, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr] + if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }} + name: Prepare remote runner test image + runs-on: ubuntu-latest + environment: integration + permissions: + actions: read + checks: write + pull-requests: write + id-token: write + contents: read + outputs: + remote-runner-version: ${{ steps.set-remote-runner-version.outputs.remote-runner-version }} + env: + ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Build Test Runner Image + uses: ./.github/actions/build-test-image + if: ${{ inputs.with_existing_remote_runner_version == '' }} + with: + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + - name: Set Remote Runner Version + id: set-remote-runner-version + run: | + if [[ -z "${{ inputs.with_existing_remote_runner_version }}" ]]; then + echo "remote-runner-version=${{ github.sha }}" >> $GITHUB_OUTPUT + else + echo "remote-runner-version=${{ inputs.with_existing_remote_runner_version }}" >> $GITHUB_OUTPUT + fi + + run-k8s-runner-tests: + needs: [load-test-configurations, prepare-remote-runner-test-image, require-chainlink-image-versions-in-qa-ecr, require-chainlink-plugin-versions-in-qa-ecr, get_latest_chainlink_release_version] + if: ${{ needs.load-test-configurations.outputs.run-k8s-tests == 'true' && always() && !failure() && !cancelled() }} + name: Run k8s tests (${{ matrix.tests.id }}) + runs-on: ${{ matrix.tests.runs_on }} + strategy: + fail-fast: false + matrix: ${{fromJson(needs.load-test-configurations.outputs.k8s-runner-matrix)}} + environment: integration + permissions: + actions: read + checks: write + pull-requests: write + id-token: write + contents: read + env: + LATEST_CHAINLINK_RELEASE_VERSION: ${{ needs.get_latest_chainlink_release_version.outputs.latest_chainlink_release_version }} + steps: + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 + with: + id: e2e_tests_${{ matrix.tests.id_sanitized }} + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Run E2E Tests / Run k8s tests (${{ matrix.tests.id }}) + continue-on-error: true + + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Install jq + run: sudo apt-get install -y jq + - name: Show Test Configuration + run: echo '${{ toJson(matrix.tests) }}' | jq . + - name: Show Remote Runner Version + run: | + echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}" + + - name: Run tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 + env: + DETACH_RUNNER: true + RR_MEM: ${{ matrix.tests.remote_runner_memory }} + TEST_ARGS: -test.timeout 900h -test.memprofile memprofile.out -test.cpuprofile profile.out + ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }} + INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com + # We can comment these out when we have a stable soak test and aren't worried about resource consumption + TEST_UPLOAD_CPU_PROFILE: true + TEST_UPLOAD_MEM_PROFILE: true + TEST_LOG_LEVEL: debug + REF_NAME: ${{ github.head_ref || github.ref_name }} + with: + test_command_to_run: ${{ matrix.tests.test_cmd }} + test_download_vendor_packages_command: make gomod + test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }} + # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 + # test_config_override_base64: ${{ inputs.test_config_override_base64 }} + test_config_chainlink_version: ${{ matrix.tests.test_inputs.chainlink_version || inputs.chainlink_version || github.sha }} + test_config_chainlink_upgrade_version: ${{ matrix.tests.test_inputs.chainlink_upgrade_version }} + test_config_chainlink_postgres_version: ${{ matrix.tests.test_inputs.chainlink_postgres_version }} + test_config_selected_networks: ${{ matrix.tests.test_inputs.selected_networks || env.SELECTED_NETWORKS}} + test_config_logging_run_id: ${{ github.run_id }} + test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + test_type: ${{ matrix.tests.test_inputs.test_type }} + test_suite: ${{ matrix.tests.test_inputs.test_suite }} + token: ${{ secrets.GH_TOKEN }} + should_cleanup: false + no_cache: true # Do not restore cache since go was already configured in the previous step + go_mod_path: ./integration-tests/go.mod + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + DEFAULT_CHAINLINK_IMAGE: ${{ matrix.tests.test_inputs.chainlink_image || env.CHAINLINK_IMAGE }} + DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ matrix.tests.test_inputs.chainlink_upgrade_image }} + DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" + DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} + DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} + DEFAULT_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} + DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} + + after_tests: + needs: [run-docker-tests, run-k8s-runner-tests] + if: always() + name: After tests notifications + runs-on: ubuntu-latest + steps: + - name: Determine combined test results + id: combine_results + run: | + docker_result="${{ needs.run-docker-tests.result }}" + k8s_result="${{ needs.run-k8s-runner-tests.result }}" + + function map_outcome { + case "$1" in + success|skipped) + echo "success" + ;; + cancelled) + echo "cancelled" + ;; + *) + echo "failure" + ;; + esac + } + + combined_docker_result=$(map_outcome $docker_result) + combined_k8s_result=$(map_outcome $k8s_result) + + if [[ $combined_docker_result == "failure" || $combined_k8s_result == "failure" ]]; then + echo "result=failure" >> $GITHUB_OUTPUT + elif [[ $combined_docker_result == "cancelled" || $combined_k8s_result == "cancelled" ]]; then + echo "result=cancelled" >> $GITHUB_OUTPUT + else + echo "result=success" >> $GITHUB_OUTPUT + fi + + - name: Send Slack notification + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + if: ${{ inputs.slack_notification_after_tests }} + id: slack + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + with: + channel-id: ${{ inputs.slack_notification_after_tests_channel_id }} + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ inputs.slack_notification_after_tests_name }} - ${{ steps.combine_results.outputs.result == 'failure' && 'Failed :x:' || steps.combine_results.outputs.result == 'cancelled' && 'Cancelled :warning:' || 'Passed :white_check_mark:' }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Build Details>" + } + } + ] + } + + # Run K8s tests using new remote runner + # remote-runner-k8s-tests: + # runs-on: ubuntu-latest + # container: + # image: golang:1.18 + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 + + # - name: Set up Go + # uses: actions/setup-go@v2 + # with: + # go-version: '1.18' + + # - name: Load Runner Config + # run: echo "$RUNNER_CONFIG" > runner.toml + # env: + # RUNNER_CONFIG: | + # # Runner configuration + # detached_mode = true + # debug = false + + # [[test_runs]] + # namespace = "dev-env" + # rbac_role_name = "dev-role" + # rbac_service_account_name = "dev-service-account" + # sync_value = "unique-sync-value-1" + # ttl_seconds_after_finished = 300 + # image_registry_url = "https://myregistry.dev/" + # image_name = "dev-image" + # image_tag = "v1.0.0" + # test_name = "TestMercuryLoad/all_endpoints" + # test_config_base64_env_name = "CONFIG_ENV_DEV" + # test_config_file_path = "/configs/dev/test-config.toml" + # test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgZGV2" + # test_timeout = "30m" + # resources_requests_cpu = "500m" + # resources_requests_memory = "1Gi" + # resources_limits_cpu = "1000m" + # resources_limits_memory = "2Gi" + # job_count = 2 + # chart_path = "/charts/dev" + # [envs] + # WASP_LOG_LEVEL = "info" + # TEST_LOG_LEVEL = "info" + # MERCURY_TEST_LOG_LEVEL = "info" + + # [[test_runs]] + # namespace = "prod-env" + # rbac_role_name = "prod-role" + # rbac_service_account_name = "prod-service-account" + # sync_value = "unique-sync-value-2" + # ttl_seconds_after_finished = 600 + # image_registry_url = "https://myregistry.prod/" + # image_name = "prod-image" + # image_tag = "v1.0.1" + # test_name = "TestMercuryLoad/all_endpoints" + # test_config_base64_env_name = "CONFIG_ENV_PROD" + # test_config_file_path = "/configs/prod/test-config.toml" + # test_config_base64 = "dGVzdCBjb25maWcgdmFsdWUgcHJvZA==" + # test_timeout = "45m" + # resources_requests_cpu = "800m" + # resources_requests_memory = "2Gi" + # resources_limits_cpu = "1500m" + # resources_limits_memory = "4Gi" + # job_count = 3 + # chart_path = "/charts/prod" + # [envs] + # WASP_LOG_LEVEL = "info" + # TEST_LOG_LEVEL = "info" + # MERCURY_TEST_LOG_LEVEL = "info" + + # # Schedule the tests in K8s in remote runner + # - name: Run Kubernetes Tests + # run: go run ./cmd/main.go run -c runner.toml \ No newline at end of file diff --git a/.github/workflows/run-nightly-e2e-tests.yml b/.github/workflows/run-nightly-e2e-tests.yml new file mode 100644 index 00000000000..ab12b36555f --- /dev/null +++ b/.github/workflows/run-nightly-e2e-tests.yml @@ -0,0 +1,34 @@ +name: Run Nightly E2E Tests + +on: + schedule: + # Run every night at midnight UTC (0:00 AM) + - cron: '0 0 * * *' + workflow_dispatch: + +jobs: + call-run-e2e-tests-workflow: + name: Run E2E Tests + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + chainlink_version: develop + test_workflow: Run Nightly E2E Tests + slack_notification_after_tests: true + slack_notification_after_tests_channel_id: "#team-test-tooling-internal" + slack_notification_after_tests_name: Nightly E2E Tests + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} diff --git a/.github/workflows/run-selected-e2e-tests.yml b/.github/workflows/run-selected-e2e-tests.yml new file mode 100644 index 00000000000..ab064531fd3 --- /dev/null +++ b/.github/workflows/run-selected-e2e-tests.yml @@ -0,0 +1,70 @@ +name: Run Selected E2E Tests + +on: + workflow_dispatch: + inputs: + chainlink_version: + description: 'Enter Chainlink version to use for the tests. Example: "v2.10.0" or sha' + required: false + type: string + test_ids: + description: 'Run all tests "*" by default. Or, enter test IDs to run separated by commas. Example: "run_all_in_ocr_tests_go,run_TestOCRv2Request_in_ocr2_test_go". Check all test IDs in .github/e2e-tests.yml' + default: "*" + required: true + type: string + test_secrets_override_key: + description: 'Enter the secret key to override test secrets' + required: false + type: string + # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 + # test_config_override_base64: + # required: false + # description: The base64-encoded test config override + # type: string + enable_check_test_configurations: + description: 'Set to "true" to enable check-test-configurations job' + required: false + type: boolean + default: false + with_existing_remote_runner_version: + description: 'Use the existing remote runner version for k8s tests. Example: "d3bf5044af33e08be788a2df31c4a745cf69d787"' + required: false + type: string + workflow_run_name: + description: 'Enter the name of the workflow run' + default: 'Run E2E Tests' + required: false + type: string + +run-name: ${{ inputs.workflow_run_name }} + +jobs: + call-run-e2e-tests-workflow: + name: Run E2E Tests + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + chainlink_version: ${{ github.event.inputs.chainlink_version }} + test_ids: ${{ github.event.inputs.test_ids }} + # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 + # test_config_override_base64: ${{ github.event.inputs.test_config_override_base64 }} + with_existing_remote_runner_version: ${{ github.event.inputs.with_existing_remote_runner_version }} + # Use fromJSON to convert string to boolean. More info: https://github.com/actions/runner/issues/2206#issuecomment-1532246677 + enable_check_test_configurations: ${{ fromJSON(github.event.inputs.enable_check_test_configurations) }} + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }} + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + diff --git a/.gitignore b/.gitignore index 4f07e40516c..2b31c9d3a59 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,7 @@ __debug_bin* integration-tests/**/traces/ benchmark_report.csv benchmark_summary.json +integration-tests/citool/output.csv # goreleaser builds cosign.* diff --git a/integration-tests/citool/cmd/check_tests_cmd.go b/integration-tests/citool/cmd/check_tests_cmd.go new file mode 100644 index 00000000000..3ef3712a572 --- /dev/null +++ b/integration-tests/citool/cmd/check_tests_cmd.go @@ -0,0 +1,166 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +type JobConfig struct { + Jobs map[string]struct { + Strategy struct { + Matrix struct { + Test []struct { + Path string `yaml:"path"` + TestOpts string `yaml:"testOpts"` + } `yaml:"test"` + } `yaml:"matrix"` + } `yaml:"strategy"` + } `yaml:"jobs"` +} + +var checkTestsCmd = &cobra.Command{ + Use: "check-tests [directory] [yaml file]", + Short: "Check if all tests in a directory are included in the test configurations YAML file", + Args: cobra.ExactArgs(2), + Run: func(_ *cobra.Command, args []string) { + directory := args[0] + yamlFile := args[1] + excludedDirs := []string{"../../citool"} + tests, err := extractTests(directory, excludedDirs) + if err != nil { + fmt.Println("Error extracting tests:", err) + os.Exit(1) + } + + checkTestsInPipeline(yamlFile, tests) + }, +} + +// extractTests scans the given directory and subdirectories (except the excluded ones) +// for Go test files, extracts test function names, and returns a slice of Test. +func extractTests(dir string, excludeDirs []string) ([]Test, error) { + var tests []Test + + // Resolve to absolute path + absDir, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + // filepath.WalkDir provides more control and is more efficient for skipping directories + err = filepath.WalkDir(absDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + // Check if the current path is one of the excluded directories + for _, exclude := range excludeDirs { + absExclude, _ := filepath.Abs(exclude) + if strings.HasPrefix(path, absExclude) { + if d.IsDir() { + return filepath.SkipDir // Skip this directory + } + return nil // Skip this file + } + } + + if !d.IsDir() && strings.HasSuffix(d.Name(), "_test.go") { + content, err := os.ReadFile(path) + if err != nil { + return err + } + re := regexp.MustCompile(`func (Test\w+)`) + matches := re.FindAllSubmatch(content, -1) + for _, match := range matches { + funcName := string(match[1]) + if funcName == "TestMain" { // Skip "TestMain" + continue + } + tests = append(tests, Test{ + Name: funcName, + Path: mustExtractSubpath(path, "integration-tests"), + }) + } + } + return nil + }) + + return tests, err +} + +// ExtractSubpath extracts a specific subpath from a given full path. +// If the subpath is not found, it returns an error. +func mustExtractSubpath(fullPath, subPath string) string { + index := strings.Index(fullPath, subPath) + if index == -1 { + panic("subpath not found in the provided full path") + } + return fullPath[index:] +} + +func checkTestsInPipeline(yamlFile string, tests []Test) { + data, err := os.ReadFile(yamlFile) + if err != nil { + fmt.Printf("Error reading YAML file: %s\n", err) + return + } + + var config Config + err = yaml.Unmarshal(data, &config) + if err != nil { + fmt.Printf("Error parsing YAML: %s\n", err) + return + } + + missingTests := []string{} // Track missing tests + + for _, test := range tests { + found := false + for _, item := range config.Tests { + if item.Path == test.Path { + if strings.Contains(item.TestCmd, "-test.run") { + if matchTestNameInCmd(item.TestCmd, test.Name) { + found = true + break + } + } else { + found = true + break + } + } + } + if !found { + missingTests = append(missingTests, fmt.Sprintf("ERROR: Test '%s' in file '%s' does not have CI configuration in '%s'", test.Name, test.Path, yamlFile)) + } + } + + if len(missingTests) > 0 { + for _, missing := range missingTests { + fmt.Println(missing) + } + os.Exit(1) // Exit with a failure status + } +} + +// matchTestNameInCmd checks if the given test name matches the -test.run pattern in the command string. +func matchTestNameInCmd(cmd string, testName string) bool { + testRunRegex := regexp.MustCompile(`-test\.run ([^\s]+)`) + matches := testRunRegex.FindStringSubmatch(cmd) + if len(matches) > 1 { + // Extract the regex pattern used in the -test.run command + pattern := matches[1] + + // Escape regex metacharacters in the testName before matching + escapedTestName := regexp.QuoteMeta(testName) + + // Check if the escaped test name matches the extracted pattern + return regexp.MustCompile(pattern).MatchString(escapedTestName) + } + return false +} diff --git a/integration-tests/citool/cmd/check_tests_cmd_test.go b/integration-tests/citool/cmd/check_tests_cmd_test.go new file mode 100644 index 00000000000..4b7f50e7f06 --- /dev/null +++ b/integration-tests/citool/cmd/check_tests_cmd_test.go @@ -0,0 +1,47 @@ +package cmd + +import ( + "testing" +) + +func TestMatchTestNameInCmd(t *testing.T) { + tests := []struct { + cmd string + testName string + expected bool + }{ + {"go test -test.run ^TestExample$", "TestExample", true}, + {"go test -test.run ^TestExample$", "TestAnother", false}, + {"go test -test.run ^TestExample$ -v", "TestExample", true}, + {"go test -test.run ^TestExamplePart$", "TestExample", false}, + {"go test -test.run ^TestWithNumbers123$", "TestWithNumbers123", true}, + {"go test -test.run ^Test_With_Underscores$", "Test_With_Underscores", true}, + {"go test -test.run ^Test-With-Dash$", "Test-With-Dash", true}, + {"go test -test.run ^TestWithSpace Space$", "TestWithSpace Space", true}, + {"go test -test.run ^TestWithNewline\nNewline$", "TestWithNewline\nNewline", true}, + {"go test -test.run ^TestOne$|^TestTwo$", "TestOne", true}, + {"go test -test.run ^TestOne$|^TestTwo$", "TestTwo", true}, + {"go test -test.run ^TestOne$|^TestTwo$", "TestThree", false}, + {"go test -test.run TestOne|TestTwo", "TestTwo", true}, + {"go test -test.run TestOne|TestTwo", "TestOne", true}, + {"go test -test.run TestOne|TestTwo|TestThree", "TestFour", false}, + {"go test -test.run ^TestOne$|TestTwo$", "TestTwo", true}, + {"go test -test.run ^TestOne$|TestTwo|TestThree$", "TestThree", true}, + {"go test -test.run TestOne|TestTwo|TestThree", "TestOne", true}, + {"go test -test.run TestOne|TestTwo|TestThree", "TestThree", true}, + {"go test -test.run ^TestA$|^TestB$|^TestC$", "TestA", true}, + {"go test -test.run ^TestA$|^TestB$|^TestC$", "TestB", true}, + {"go test -test.run ^TestA$|^TestB$|^TestC$", "TestD", false}, + {"go test -test.run TestA|^TestB$|TestC", "TestB", true}, + {"go test -test.run ^TestA|^TestB|TestC$", "TestA", true}, + {"go test -test.run ^TestA|^TestB|TestC$", "TestC", true}, + {"go test -test.run ^TestA|^TestB|TestC$", "TestD", false}, + } + + for _, tt := range tests { + result := matchTestNameInCmd(tt.cmd, tt.testName) + if result != tt.expected { + t.Errorf("matchTestNameInCmd(%s, %s) = %t; expected %t", tt.cmd, tt.testName, result, tt.expected) + } + } +} diff --git a/integration-tests/citool/cmd/create_test_config_cmd.go b/integration-tests/citool/cmd/create_test_config_cmd.go new file mode 100644 index 00000000000..bc1b65bcdcd --- /dev/null +++ b/integration-tests/citool/cmd/create_test_config_cmd.go @@ -0,0 +1,178 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/pelletier/go-toml/v2" + "github.com/spf13/cobra" + + ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" +) + +var createTestConfigCmd = &cobra.Command{ + Use: "create", + Short: "Create a test config from the provided flags", + Run: func(cmd *cobra.Command, _ []string) { + var tc ctf_config.TestConfig + + var version, postgresVersion *string + if cmd.Flags().Changed(ChainlinkVersionFlag) { + version = &oc.ChainlinkVersion + } + if cmd.Flags().Changed(ChainlinkPostgresVersionFlag) { + version = &oc.ChainlinkPostgresVersion + } + if version != nil || postgresVersion != nil { + tc.ChainlinkImage = &ctf_config.ChainlinkImageConfig{ + Version: version, + PostgresVersion: postgresVersion, + } + } + + var upgradeVersion *string + if cmd.Flags().Changed(ChainlinkUpgradeVersionFlag) { + upgradeVersion = &oc.ChainlinkUpgradeVersion + } + if upgradeVersion != nil { + tc.ChainlinkUpgradeImage = &ctf_config.ChainlinkImageConfig{ + Version: upgradeVersion, + } + } + + var selectedNetworks *[]string + if cmd.Flags().Changed(SelectedNetworksFlag) { + selectedNetworks = &oc.SelectedNetworks + } + if selectedNetworks != nil { + tc.Network = &ctf_config.NetworkConfig{ + SelectedNetworks: oc.SelectedNetworks, + } + } + + var peryscopeEnabled *bool + var pyroscopeServerURL, pyroscopeEnvironment, pyroscopeKey *string + if cmd.Flags().Changed(PyroscopeEnabledFlag) { + peryscopeEnabled = &oc.PyroscopeEnabled + } + if cmd.Flags().Changed(PyroscopeServerURLFlag) { + pyroscopeServerURL = &oc.PyroscopeServerURL + } + if cmd.Flags().Changed(PyroscopeKeyFlag) { + pyroscopeKey = &oc.PyroscopeKey + } + if cmd.Flags().Changed(PyroscopeEnvironmentFlag) { + pyroscopeEnvironment = &oc.PyroscopeEnvironment + } + if peryscopeEnabled != nil { + tc.Pyroscope = &ctf_config.PyroscopeConfig{ + Enabled: peryscopeEnabled, + ServerUrl: pyroscopeServerURL, + Environment: pyroscopeEnvironment, + Key: pyroscopeKey, + } + } + + var testLogCollect *bool + if cmd.Flags().Changed(LoggingTestLogCollectFlag) { + testLogCollect = &oc.LoggingTestLogCollect + } + var loggingRunID *string + if cmd.Flags().Changed(LoggingRunIDFlag) { + loggingRunID = &oc.LoggingRunID + } + var loggingLogTargets []string + if cmd.Flags().Changed(LoggingLogTargetsFlag) { + loggingLogTargets = oc.LoggingLogTargets + } + var loggingLokiTenantID *string + if cmd.Flags().Changed(LoggingLokiTenantIDFlag) { + loggingLokiTenantID = &oc.LoggingLokiTenantID + } + var loggingLokiBasicAuth *string + if cmd.Flags().Changed(LoggingLokiBasicAuthFlag) { + loggingLokiBasicAuth = &oc.LoggingLokiBasicAuth + } + var loggingLokiEndpoint *string + if cmd.Flags().Changed(LoggingLokiEndpointFlag) { + loggingLokiEndpoint = &oc.LoggingLokiEndpoint + } + var loggingGrafanaBaseURL *string + if cmd.Flags().Changed(LoggingGrafanaBaseURLFlag) { + loggingGrafanaBaseURL = &oc.LoggingGrafanaBaseURL + } + var loggingGrafanaDashboardURL *string + if cmd.Flags().Changed(LoggingGrafanaDashboardURLFlag) { + loggingGrafanaDashboardURL = &oc.LoggingGrafanaDashboardURL + } + var loggingGrafanaBearerToken *string + if cmd.Flags().Changed(LoggingGrafanaBearerTokenFlag) { + loggingGrafanaBearerToken = &oc.LoggingGrafanaBearerToken + } + + if testLogCollect != nil || loggingRunID != nil || loggingLogTargets != nil || loggingLokiEndpoint != nil || loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaBearerToken != nil { + tc.Logging = &ctf_config.LoggingConfig{} + tc.Logging.TestLogCollect = testLogCollect + tc.Logging.RunId = loggingRunID + if loggingLogTargets != nil { + tc.Logging.LogStream = &ctf_config.LogStreamConfig{ + LogTargets: loggingLogTargets, + } + } + if loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingLokiEndpoint != nil { + tc.Logging.Loki = &ctf_config.LokiConfig{ + TenantId: loggingLokiTenantID, + BasicAuth: loggingLokiBasicAuth, + Endpoint: loggingLokiEndpoint, + } + } + if loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaBearerToken != nil { + tc.Logging.Grafana = &ctf_config.GrafanaConfig{ + BaseUrl: loggingGrafanaBaseURL, + DashboardUrl: loggingGrafanaDashboardURL, + BearerToken: loggingGrafanaBearerToken, + } + } + } + + var privateEthereumNetworkExecutionLayer *string + if cmd.Flags().Changed(PrivateEthereumNetworkExecutionLayerFlag) { + privateEthereumNetworkExecutionLayer = &oc.PrivateEthereumNetworkExecutionLayer + } + var privateEthereumNetworkEthereumVersion *string + if cmd.Flags().Changed(PrivateEthereumNetworkEthereumVersionFlag) { + privateEthereumNetworkEthereumVersion = &oc.PrivateEthereumNetworkEthereumVersion + } + var privateEthereumNetworkCustomDockerImage *string + if cmd.Flags().Changed(PrivateEthereumNetworkCustomDockerImageFlag) { + privateEthereumNetworkCustomDockerImage = &oc.PrivateEthereumNetworkCustomDockerImages + } + if privateEthereumNetworkExecutionLayer != nil || privateEthereumNetworkEthereumVersion != nil || privateEthereumNetworkCustomDockerImage != nil { + var el ctf_config.ExecutionLayer + if privateEthereumNetworkExecutionLayer != nil { + el = ctf_config.ExecutionLayer(*privateEthereumNetworkExecutionLayer) + } + var ev ctf_config.EthereumVersion + if privateEthereumNetworkEthereumVersion != nil { + ev = ctf_config.EthereumVersion(*privateEthereumNetworkEthereumVersion) + } + var customImages map[ctf_config.ContainerType]string + if privateEthereumNetworkCustomDockerImage != nil { + customImages = map[ctf_config.ContainerType]string{"execution_layer": *privateEthereumNetworkCustomDockerImage} + } + tc.PrivateEthereumNetwork = &ctf_config.EthereumNetworkConfig{ + ExecutionLayer: &el, + EthereumVersion: &ev, + CustomDockerImages: customImages, + } + } + + configToml, err := toml.Marshal(tc) + if err != nil { + fmt.Fprintf(os.Stderr, "Error marshalling TestConfig to TOML: %v\n", err) + os.Exit(1) + } + + fmt.Fprintln(cmd.OutOrStdout(), string(configToml)) + }, +} diff --git a/integration-tests/citool/cmd/csv_export_cmd.go b/integration-tests/citool/cmd/csv_export_cmd.go new file mode 100644 index 00000000000..8fe13440c81 --- /dev/null +++ b/integration-tests/citool/cmd/csv_export_cmd.go @@ -0,0 +1,96 @@ +package cmd + +import ( + "encoding/csv" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +var csvExportCmd = &cobra.Command{ + Use: "csvexport", + Short: "Export tests to CSV format", + Run: func(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString("file") + if err := exportConfigToCSV(configFile); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + }, +} + +func init() { + csvExportCmd.Flags().StringP("file", "f", "", "Path to YML file") + err := csvExportCmd.MarkFlagRequired("file") + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +func exportConfigToCSV(configFile string) error { + // Read the YAML file + bytes, err := os.ReadFile(configFile) + if err != nil { + return err + } + + // Unmarshal the YAML into the Config struct + var config Config + if err := yaml.Unmarshal(bytes, &config); err != nil { + return err + } + + // Create a CSV file + file, err := os.Create("output.csv") + if err != nil { + return err + } + defer file.Close() + + writer := csv.NewWriter(file) + defer writer.Flush() + + // Write CSV headers + headers := []string{"ID", "Test Path", "Test Env Type", "Runs On", "Test Cmd", "Test Config Override Required", "Test Secrets Required", "Remote Runner Memory", "Pyroscope Env", "Workflows", "Test Inputs"} + if err := writer.Write(headers); err != nil { + return err + } + + // Iterate over Tests and write data to CSV + for _, test := range config.Tests { + workflows := strings.Join(test.Workflows, ", ") // Combine workflows into a single CSV field + // Serialize TestInputs + testInputs := serializeMap(test.TestInputs) + + record := []string{ + test.ID, + test.Path, + test.TestEnvType, + test.RunsOn, + test.TestCmd, + fmt.Sprintf("%t", test.TestConfigOverrideRequired), + fmt.Sprintf("%t", test.TestSecretsRequired), + test.RemoteRunnerMemory, + test.PyroscopeEnv, + workflows, + testInputs, + } + if err := writer.Write(record); err != nil { + return err + } + } + + return nil +} + +func serializeMap(inputs map[string]string) string { + pairs := make([]string, 0, len(inputs)) + for key, value := range inputs { + pairs = append(pairs, fmt.Sprintf("%s=%s", key, value)) + } + return strings.Join(pairs, ", ") +} diff --git a/integration-tests/citool/cmd/filter_cmd.go b/integration-tests/citool/cmd/filter_cmd.go new file mode 100644 index 00000000000..c1d5f22357d --- /dev/null +++ b/integration-tests/citool/cmd/filter_cmd.go @@ -0,0 +1,165 @@ +package cmd + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "log" + "os" + "regexp" + "strings" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" + + "github.com/smartcontractkit/chainlink-testing-framework/utils" +) + +// Filter tests based on workflow, test type, and test IDs. +func filterTests(allTests []CITestConf, workflow, testType, ids string, envresolve bool) []CITestConf { + workflowFilter := workflow + typeFilter := testType + idFilter := strings.Split(ids, ",") + + var filteredTests []CITestConf + + for _, test := range allTests { + workflowMatch := workflow == "" || contains(test.Workflows, workflowFilter) + typeMatch := testType == "" || test.TestEnvType == typeFilter + idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID) + + if workflowMatch && typeMatch && idMatch { + test.IDSanitized = sanitizeTestID(test.ID) + filteredTests = append(filteredTests, test) + } + if envresolve { + for k, v := range test.TestInputs { + test.TestInputs[k] = utils.MustResolveEnvPlaceholder(v) + } + } + } + + return filteredTests +} + +func filterAndMergeTests(allTests []CITestConf, workflow, testType, base64Tests string, envresolve bool) ([]CITestConf, error) { + decodedBytes, err := base64.StdEncoding.DecodeString(base64Tests) + if err != nil { + return nil, err + } + var decodedTests []CITestConf + err = yaml.Unmarshal(decodedBytes, &decodedTests) + if err != nil { + return nil, err + } + + idFilter := make(map[string]CITestConf) + for _, dt := range decodedTests { + idFilter[dt.ID] = dt + } + + var filteredTests []CITestConf + for _, test := range allTests { + workflowMatch := workflow == "" || contains(test.Workflows, workflow) + typeMatch := testType == "" || test.TestEnvType == testType + + if decodedTest, exists := idFilter[test.ID]; exists && workflowMatch && typeMatch { + // Override test inputs from the base64 encoded tests + for k, v := range decodedTest.TestInputs { + if test.TestInputs == nil { + test.TestInputs = make(map[string]string) + } + test.TestInputs[k] = v + } + test.IDSanitized = sanitizeTestID(test.ID) + filteredTests = append(filteredTests, test) + } + if envresolve { + for k, v := range test.TestInputs { + test.TestInputs[k] = utils.MustResolveEnvPlaceholder(v) + } + } + } + + return filteredTests, nil +} + +func sanitizeTestID(id string) string { + // Define a regular expression that matches any character not a letter, digit, hyphen + re := regexp.MustCompile(`[^a-zA-Z0-9-_]+`) + // Replace all occurrences of disallowed characters with "_" + return re.ReplaceAllString(id, "_") +} + +// Utility function to check if a slice contains a string. +func contains(slice []string, element string) bool { + for _, s := range slice { + if s == element { + return true + } + } + return false +} + +// filterCmd represents the filter command +var filterCmd = &cobra.Command{ + Use: "filter", + Short: "Filter test configurations based on specified criteria", + Long: `Filters tests from a YAML configuration based on name, workflow, test type, and test IDs. +Example usage: +./e2e_tests_tool filter --file .github/e2e-tests.yml --workflow "Run Nightly E2E Tests" --test-env-type "docker" --test-ids "test1,test2"`, + Run: func(cmd *cobra.Command, _ []string) { + yamlFile, _ := cmd.Flags().GetString("file") + workflow, _ := cmd.Flags().GetString("workflow") + testType, _ := cmd.Flags().GetString("test-env-type") + testIDs, _ := cmd.Flags().GetString("test-ids") + testMap, _ := cmd.Flags().GetString("test-list") + envresolve, _ := cmd.Flags().GetBool("envresolve") + + data, err := os.ReadFile(yamlFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error reading YAML file: %v\n", err) + os.Exit(1) + } + + var config Config + err = yaml.Unmarshal(data, &config) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing YAML file %s data: %v\n", yamlFile, err) + os.Exit(1) + } + + var filteredTests []CITestConf + if testMap == "" { + filteredTests = filterTests(config.Tests, workflow, testType, testIDs, envresolve) + } else { + filteredTests, err = filterAndMergeTests(config.Tests, workflow, testType, testMap, envresolve) + if err != nil { + log.Fatalf("Error filtering and merging tests: %v", err) + } + } + matrix := map[string][]CITestConf{"tests": filteredTests} + matrixJSON, err := json.Marshal(matrix) + if err != nil { + fmt.Fprintf(os.Stderr, "Error marshaling matrix to JSON: %v\n", err) + os.Exit(1) + } + + fmt.Printf("%s", matrixJSON) + }, +} + +func init() { + filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file") + filterCmd.Flags().String("test-list", "", "Base64 encoded list of tests (YML objects) to filter by. Can include test_inputs for each test.") + filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by") + filterCmd.Flags().StringP("test-env-type", "y", "", "Type of test to filter by") + filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter") + filterCmd.Flags().Bool("envresolve", false, "Resolve environment variables in test inputs") + + err := filterCmd.MarkFlagRequired("file") + if err != nil { + fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err) + os.Exit(1) + } +} diff --git a/integration-tests/citool/cmd/filter_cmd_test.go b/integration-tests/citool/cmd/filter_cmd_test.go new file mode 100644 index 00000000000..ff6e9c981de --- /dev/null +++ b/integration-tests/citool/cmd/filter_cmd_test.go @@ -0,0 +1,64 @@ +package cmd + +import ( + "testing" +) + +func TestFilterTestsByID(t *testing.T) { + tests := []CITestConf{ + {ID: "run_all_in_ocr_tests_go", TestEnvType: "docker"}, + {ID: "run_all_in_ocr2_tests_go", TestEnvType: "docker"}, + {ID: "run_all_in_ocr3_tests_go", TestEnvType: "k8s_remote_runner"}, + } + + cases := []struct { + description string + inputIDs string + expectedLen int + }{ + {"Filter by single ID", "run_all_in_ocr_tests_go", 1}, + {"Filter by multiple IDs", "run_all_in_ocr_tests_go,run_all_in_ocr2_tests_go", 2}, + {"Wildcard to include all", "*", 3}, + {"Empty ID string to include all", "", 3}, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + filtered := filterTests(tests, "", "", c.inputIDs, false) + if len(filtered) != c.expectedLen { + t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen) + } + }) + } +} + +func TestFilterTestsIntegration(t *testing.T) { + tests := []CITestConf{ + {ID: "run_all_in_ocr_tests_go", TestEnvType: "docker", Workflows: []string{"Run Nightly E2E Tests"}}, + {ID: "run_all_in_ocr2_tests_go", TestEnvType: "docker", Workflows: []string{"Run PR E2E Tests"}}, + {ID: "run_all_in_ocr3_tests_go", TestEnvType: "k8s_remote_runner", Workflows: []string{"Run PR E2E Tests"}}, + } + + cases := []struct { + description string + inputNames string + inputWorkflow string + inputTestType string + inputIDs string + expectedLen int + }{ + {"Filter by test type and ID", "", "", "docker", "run_all_in_ocr2_tests_go", 1}, + {"Filter by trigger and test type", "", "Run PR E2E Tests", "docker", "*", 1}, + {"No filters applied", "", "", "", "*", 3}, + {"Filter mismatching all criteria", "", "Run Nightly E2E Tests", "", "", 1}, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + filtered := filterTests(tests, c.inputWorkflow, c.inputTestType, c.inputIDs, false) + if len(filtered) != c.expectedLen { + t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen) + } + }) + } +} diff --git a/integration-tests/citool/cmd/root_cmd.go b/integration-tests/citool/cmd/root_cmd.go new file mode 100644 index 00000000000..fdb4efd5724 --- /dev/null +++ b/integration-tests/citool/cmd/root_cmd.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "citool", + Short: "A tool to manage E2E tests on Github CI", +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + + rootCmd.AddCommand(checkTestsCmd) + rootCmd.AddCommand(filterCmd) + rootCmd.AddCommand(csvExportCmd) + rootCmd.AddCommand(testConfigCmd) + testConfigCmd.AddCommand(createTestConfigCmd) +} diff --git a/integration-tests/citool/cmd/test_config_cmd.go b/integration-tests/citool/cmd/test_config_cmd.go new file mode 100644 index 00000000000..0c0e272353b --- /dev/null +++ b/integration-tests/citool/cmd/test_config_cmd.go @@ -0,0 +1,123 @@ +package cmd + +import ( + "strings" + + "github.com/spf13/cobra" + + "github.com/smartcontractkit/chainlink-testing-framework/utils" +) + +var testConfigCmd = &cobra.Command{ + Use: "test-config", + Short: "Manage test config", +} + +// OverrideConfig holds the configuration data for overrides +type OverrideConfig struct { + ChainlinkImage string + ChainlinkVersion string + ChainlinkUpgradeImage string + ChainlinkUpgradeVersion string + ChainlinkPostgresVersion string + SelectedNetworks []string + PyroscopeEnabled bool + PyroscopeServerURL string + PyroscopeEnvironment string + PyroscopeKey string + LoggingTestLogCollect bool + LoggingRunID string + LoggingLogTargets []string + LoggingLokiTenantID string + LoggingLokiEndpoint string + LoggingLokiBasicAuth string + LoggingGrafanaBaseURL string + LoggingGrafanaDashboardURL string + LoggingGrafanaBearerToken string + PrivateEthereumNetworkExecutionLayer string + PrivateEthereumNetworkEthereumVersion string + PrivateEthereumNetworkCustomDockerImages string +} + +const ( + ChainlinkVersionFlag = "chainlink-version" + ChainlinkUpgradeVersionFlag = "chainlink-upgrade-version" + ChainlinkPostgresVersionFlag = "chainlink-postgres-version" + SelectedNetworksFlag = "selected-networks" + FromBase64ConfigFlag = "from-base64-config" + LoggingLokiBasicAuthFlag = "logging-loki-basic-auth" + LoggingLokiEndpointFlag = "logging-loki-endpoint" + LoggingRunIDFlag = "logging-run-id" + LoggingLokiTenantIDFlag = "logging-loki-tenant-id" + LoggingGrafanaBaseURLFlag = "logging-grafana-base-url" + LoggingGrafanaDashboardURLFlag = "logging-grafana-dashboard-url" + LoggingGrafanaBearerTokenFlag = "logging-grafana-bearer-token" + LoggingLogTargetsFlag = "logging-log-targets" + LoggingTestLogCollectFlag = "logging-test-log-collect" + PyroscopeEnabledFlag = "pyroscope-enabled" + PyroscopeServerURLFlag = "pyroscope-server-url" + PyroscopeKeyFlag = "pyroscope-key" + PyroscopeEnvironmentFlag = "pyroscope-environment" + PrivateEthereumNetworkExecutionLayerFlag = "private-ethereum-network-execution-layer" + PrivateEthereumNetworkEthereumVersionFlag = "private-ethereum-network-ethereum-version" + PrivateEthereumNetworkCustomDockerImageFlag = "private-ethereum-network-custom-docker-image" +) + +var oc OverrideConfig + +func init() { + cmds := []*cobra.Command{createTestConfigCmd} + for _, c := range cmds { + c.Flags().StringArrayVar(&oc.SelectedNetworks, SelectedNetworksFlag, nil, "Selected networks") + c.Flags().StringVar(&oc.ChainlinkVersion, ChainlinkVersionFlag, "", "Chainlink version") + c.Flags().StringVar(&oc.ChainlinkUpgradeVersion, ChainlinkUpgradeVersionFlag, "", "Chainlink upgrade version") + c.Flags().StringVar(&oc.ChainlinkPostgresVersion, ChainlinkPostgresVersionFlag, "", "Chainlink Postgres version") + c.Flags().BoolVar(&oc.PyroscopeEnabled, PyroscopeEnabledFlag, false, "Pyroscope enabled") + c.Flags().StringVar(&oc.PyroscopeServerURL, PyroscopeServerURLFlag, "", "Pyroscope server URL") + c.Flags().StringVar(&oc.PyroscopeKey, PyroscopeKeyFlag, "", "Pyroscope key") + c.Flags().StringVar(&oc.PyroscopeEnvironment, PyroscopeEnvironmentFlag, "", "Pyroscope environment") + c.Flags().BoolVar(&oc.LoggingTestLogCollect, LoggingTestLogCollectFlag, false, "Test log collect") + c.Flags().StringVar(&oc.LoggingRunID, LoggingRunIDFlag, "", "Run ID") + c.Flags().StringArrayVar(&oc.LoggingLogTargets, LoggingLogTargetsFlag, nil, "Logging.LogStream.LogTargets") + c.Flags().StringVar(&oc.LoggingLokiEndpoint, LoggingLokiEndpointFlag, "", "") + c.Flags().StringVar(&oc.LoggingLokiTenantID, LoggingLokiTenantIDFlag, "", "") + c.Flags().StringVar(&oc.LoggingLokiBasicAuth, LoggingLokiBasicAuthFlag, "", "") + c.Flags().StringVar(&oc.LoggingGrafanaBaseURL, LoggingGrafanaBaseURLFlag, "", "") + c.Flags().StringVar(&oc.LoggingGrafanaDashboardURL, LoggingGrafanaDashboardURLFlag, "", "") + c.Flags().StringVar(&oc.LoggingGrafanaBearerToken, LoggingGrafanaBearerTokenFlag, "", "") + c.Flags().StringVar(&oc.PrivateEthereumNetworkExecutionLayer, PrivateEthereumNetworkExecutionLayerFlag, "", "") + c.Flags().StringVar(&oc.PrivateEthereumNetworkEthereumVersion, PrivateEthereumNetworkEthereumVersionFlag, "", "") + c.Flags().StringVar(&oc.PrivateEthereumNetworkCustomDockerImages, PrivateEthereumNetworkCustomDockerImageFlag, "", "") + + c.PreRun = func(_ *cobra.Command, _ []string) { + // Resolve selected networks environment variable if set + if len(oc.SelectedNetworks) > 0 { + _, hasEnvVar := utils.LookupEnvVarName(oc.SelectedNetworks[0]) + if hasEnvVar { + selectedNetworks := utils.MustResolveEnvPlaceholder(oc.SelectedNetworks[0]) + oc.SelectedNetworks = strings.Split(selectedNetworks, ",") + } + } + + // Resolve all other environment variables + oc.ChainlinkImage = utils.MustResolveEnvPlaceholder(oc.ChainlinkImage) + oc.ChainlinkVersion = utils.MustResolveEnvPlaceholder(oc.ChainlinkVersion) + oc.ChainlinkUpgradeImage = utils.MustResolveEnvPlaceholder(oc.ChainlinkUpgradeImage) + oc.ChainlinkUpgradeVersion = utils.MustResolveEnvPlaceholder(oc.ChainlinkUpgradeVersion) + oc.ChainlinkPostgresVersion = utils.MustResolveEnvPlaceholder(oc.ChainlinkPostgresVersion) + oc.PyroscopeServerURL = utils.MustResolveEnvPlaceholder(oc.PyroscopeServerURL) + oc.PyroscopeKey = utils.MustResolveEnvPlaceholder(oc.PyroscopeKey) + oc.PyroscopeEnvironment = utils.MustResolveEnvPlaceholder(oc.PyroscopeEnvironment) + oc.LoggingRunID = utils.MustResolveEnvPlaceholder(oc.LoggingRunID) + oc.LoggingLokiTenantID = utils.MustResolveEnvPlaceholder(oc.LoggingLokiTenantID) + oc.LoggingLokiEndpoint = utils.MustResolveEnvPlaceholder(oc.LoggingLokiEndpoint) + oc.LoggingLokiBasicAuth = utils.MustResolveEnvPlaceholder(oc.LoggingLokiBasicAuth) + oc.LoggingGrafanaBaseURL = utils.MustResolveEnvPlaceholder(oc.LoggingGrafanaBaseURL) + oc.LoggingGrafanaDashboardURL = utils.MustResolveEnvPlaceholder(oc.LoggingGrafanaDashboardURL) + oc.LoggingGrafanaBearerToken = utils.MustResolveEnvPlaceholder(oc.LoggingGrafanaBearerToken) + oc.PrivateEthereumNetworkExecutionLayer = utils.MustResolveEnvPlaceholder(oc.PrivateEthereumNetworkExecutionLayer) + oc.PrivateEthereumNetworkEthereumVersion = utils.MustResolveEnvPlaceholder(oc.PrivateEthereumNetworkEthereumVersion) + oc.PrivateEthereumNetworkCustomDockerImages = utils.MustResolveEnvPlaceholder(oc.PrivateEthereumNetworkCustomDockerImages) + } + } +} diff --git a/integration-tests/citool/cmd/test_config_cmd_test.go b/integration-tests/citool/cmd/test_config_cmd_test.go new file mode 100644 index 00000000000..fb1ef5332bd --- /dev/null +++ b/integration-tests/citool/cmd/test_config_cmd_test.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "bytes" + "testing" + + "github.com/pelletier/go-toml/v2" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + + ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" +) + +func TestCreateTestConfigCmd(t *testing.T) { + tests := []struct { + name string + args []string + want interface{} + check func(t *testing.T, tc *ctf_config.TestConfig) + wantErr bool + }{ + { + name: "LoggingLogTargets", + args: []string{"create", "--logging-log-targets=target1", "--logging-log-targets=target2"}, + check: func(t *testing.T, tc *ctf_config.TestConfig) { + assert.NotNil(t, tc.Logging) + assert.NotNil(t, tc.Logging.LogStream) + assert.Equal(t, []string{"target1", "target2"}, tc.Logging.LogStream.LogTargets) + }, + }, + { + name: "PrivateEthereumNetworkExecutionLayerFlag", + args: []string{"create", "--private-ethereum-network-execution-layer=geth", "--private-ethereum-network-ethereum-version=1.10.0"}, + check: func(t *testing.T, tc *ctf_config.TestConfig) { + assert.NotNil(t, tc.PrivateEthereumNetwork) + assert.NotNil(t, tc.PrivateEthereumNetwork.ExecutionLayer) + assert.Equal(t, ctf_config.ExecutionLayer("geth"), *tc.PrivateEthereumNetwork.ExecutionLayer) + assert.Equal(t, ctf_config.EthereumVersion("1.10.0"), *tc.PrivateEthereumNetwork.EthereumVersion) + }, + }, + { + name: "PrivateEthereumNetworkCustomDockerImageFlag", + args: []string{"create", "--private-ethereum-network-execution-layer=geth", "--private-ethereum-network-ethereum-version=1.10.0", "--private-ethereum-network-custom-docker-image=custom-image:v1.0"}, + check: func(t *testing.T, tc *ctf_config.TestConfig) { + assert.NotNil(t, tc.PrivateEthereumNetwork) + assert.NotNil(t, tc.PrivateEthereumNetwork.ExecutionLayer) + assert.Equal(t, map[ctf_config.ContainerType]string{"execution_layer": "custom-image:v1.0"}, tc.PrivateEthereumNetwork.CustomDockerImages) + }, + }, + } + + rootCmd := &cobra.Command{} + rootCmd.AddCommand(createTestConfigCmd) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rootCmd.SetArgs(tt.args) + var out bytes.Buffer + rootCmd.SetOutput(&out) + err := rootCmd.Execute() + if (err != nil) != tt.wantErr { + t.Fatalf("Execute() error = %v, wantErr %v", err, tt.wantErr) + } + var tc ctf_config.TestConfig + err = toml.Unmarshal(out.Bytes(), &tc) + if err != nil { + t.Fatalf("Failed to unmarshal output: %v", err) + } + if tt.check != nil { + tt.check(t, &tc) + } + }) + } +} diff --git a/integration-tests/citool/cmd/types.go b/integration-tests/citool/cmd/types.go new file mode 100644 index 00000000000..3c347e9406b --- /dev/null +++ b/integration-tests/citool/cmd/types.go @@ -0,0 +1,26 @@ +package cmd + +type Test struct { + Name string + Path string +} + +// CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows. +type CITestConf struct { + ID string `yaml:"id" json:"id"` + IDSanitized string `json:"id_sanitized"` + Path string `yaml:"path" json:"path"` + TestEnvType string `yaml:"test_env_type" json:"test_env_type"` + RunsOn string `yaml:"runs_on" json:"runs_on"` + TestCmd string `yaml:"test_cmd" json:"test_cmd"` + TestConfigOverrideRequired bool `yaml:"test_config_override_required" json:"testConfigOverrideRequired"` + TestSecretsRequired bool `yaml:"test_secrets_required" json:"testSecretsRequired"` + TestInputs map[string]string `yaml:"test_inputs" json:"test_inputs"` + RemoteRunnerMemory string `yaml:"remote_runner_memory" json:"remoteRunnerMemory"` + PyroscopeEnv string `yaml:"pyroscope_env" json:"pyroscopeEnv"` + Workflows []string `yaml:"workflows" json:"workflows"` +} + +type Config struct { + Tests []CITestConf `yaml:"runner-test-matrix"` +} diff --git a/integration-tests/citool/main.go b/integration-tests/citool/main.go new file mode 100644 index 00000000000..4fa6cac56e5 --- /dev/null +++ b/integration-tests/citool/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/smartcontractkit/chainlink/integration-tests/citool/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 411fcb37575..d65f49405fb 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -29,7 +29,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240712162033-89bd3351ce6e - github.com/smartcontractkit/chainlink-testing-framework v1.32.5 + github.com/smartcontractkit/chainlink-testing-framework v1.32.7 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 @@ -45,6 +45,8 @@ require ( golang.org/x/sync v0.7.0 golang.org/x/text v0.16.0 gopkg.in/guregu/null.v4 v4.0.0 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.28.2 ) @@ -463,8 +465,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.28.2 // indirect k8s.io/apiextensions-apiserver v0.28.1 // indirect k8s.io/cli-runtime v0.28.2 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 3a4f31131f7..07ad70c4d30 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1493,8 +1493,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240712132946-267a37c5ac6 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240712132946-267a37c5ac6e/go.mod h1:hsFhop+SlQHKD+DEFjZrMJmbauT1A/wvtZIeeo4PxFU= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= -github.com/smartcontractkit/chainlink-testing-framework v1.32.5 h1:ewwrI+93a+//7fxhc682TK3O5npqngx/3JDQdEzectY= -github.com/smartcontractkit/chainlink-testing-framework v1.32.5/go.mod h1:Y1D6k7KLPZ52kwp3WJxShp4Wzw22jKldIzMT2yosipI= +github.com/smartcontractkit/chainlink-testing-framework v1.32.7 h1:/I6Upq9KdnleWnUF1W3c3mAgMowAgi0yAcn8Vh5Px50= +github.com/smartcontractkit/chainlink-testing-framework v1.32.7/go.mod h1:Y1D6k7KLPZ52kwp3WJxShp4Wzw22jKldIzMT2yosipI= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 219308e2ddb..b1d2dbb9615 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240712162033-89bd3351ce6e - github.com/smartcontractkit/chainlink-testing-framework v1.32.5 + github.com/smartcontractkit/chainlink-testing-framework v1.32.7 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e @@ -380,7 +380,7 @@ require ( github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.15.0 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 4355312bef0..2a56fad1aa4 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -374,8 +374,8 @@ github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= @@ -1475,8 +1475,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240712132946-267a37c5ac6 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240712132946-267a37c5ac6e/go.mod h1:hsFhop+SlQHKD+DEFjZrMJmbauT1A/wvtZIeeo4PxFU= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= -github.com/smartcontractkit/chainlink-testing-framework v1.32.5 h1:ewwrI+93a+//7fxhc682TK3O5npqngx/3JDQdEzectY= -github.com/smartcontractkit/chainlink-testing-framework v1.32.5/go.mod h1:Y1D6k7KLPZ52kwp3WJxShp4Wzw22jKldIzMT2yosipI= +github.com/smartcontractkit/chainlink-testing-framework v1.32.7 h1:/I6Upq9KdnleWnUF1W3c3mAgMowAgi0yAcn8Vh5Px50= +github.com/smartcontractkit/chainlink-testing-framework v1.32.7/go.mod h1:Y1D6k7KLPZ52kwp3WJxShp4Wzw22jKldIzMT2yosipI= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= @@ -1515,8 +1515,8 @@ github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cA github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= diff --git a/integration-tests/reorg/automation_reorg_test.go b/integration-tests/reorg/automation_reorg_test.go index a393513ac49..383c8d5c95b 100644 --- a/integration-tests/reorg/automation_reorg_test.go +++ b/integration-tests/reorg/automation_reorg_test.go @@ -7,6 +7,7 @@ import ( "regexp" "strconv" "strings" + "sync" "testing" "time" @@ -59,6 +60,7 @@ var ( }, }, } + mutex sync.Mutex defaultOCRRegistryConfig = contracts.KeeperRegistrySettings{ PaymentPremiumPPB: uint32(200000000), FlatFeeMicroLINK: uint32(0), @@ -139,8 +141,10 @@ func TestAutomationReorg(t *testing.T) { tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&config, network) require.NoError(t, err, "Error building TOML config") + mutex.Lock() defaultAutomationSettings["replicas"] = nodeCount defaultAutomationSettings["toml"] = tomlConfig + mutex.Unlock() var overrideFn = func(_ interface{}, target interface{}) { ctf_config.MustConfigOverrideChainlinkVersion(config.GetChainlinkImageConfig(), target)