diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 594cb9a4392..fe178a0df68 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -490,6 +490,7 @@ runner-test-matrix: remote_runner_memory: 4Gi test_secrets_required: true test_env_vars: + TEST_LOG_LEVEL: info TEST_SUITE: automationv2_1 pyroscope_env: automation-load-test @@ -591,7 +592,7 @@ runner-test-matrix: test_env_vars: TEST_SUITE: chaos - - id: benchmark/automation_test.go:^TestAutomationBenchmark$ + - id: benchmark/automation_test.go:TestAutomationBenchmark path: integration-tests/benchmark/automation_test.go test_env_type: k8s-remote-runner remote_runner_memory: 4Gi @@ -601,9 +602,24 @@ runner-test-matrix: test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-benchmark-automation-nightly test_env_vars: + TEST_LOG_LEVEL: info TEST_SUITE: benchmark TEST_TYPE: benchmark + - id: soak/automation_test.go:TestAutomationBenchmark + path: integration-tests/benchmark/automation_test.go + test_env_type: k8s-remote-runner + remote_runner_memory: 4Gi + runs_on: ubuntu-latest + # workflows: + # - Nightly E2E Tests + test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json + pyroscope_env: ci-benchmark-automation-nightly + test_env_vars: + TEST_LOG_LEVEL: info + TEST_SUITE: benchmark + TEST_TYPE: soak + # END: Automation tests # START: VRF tests diff --git a/.github/workflows/automation-benchmark-tests.yml b/.github/workflows/automation-benchmark-tests.yml index 59bbcc02f6a..6e446959295 100644 --- a/.github/workflows/automation-benchmark-tests.yml +++ b/.github/workflows/automation-benchmark-tests.yml @@ -14,14 +14,19 @@ on: description: Notifies test results (Not your @) required: true default: U02Q14G80TY - type: string + type: string + testType: + description: Type of test to run (benchmark, soak) + required: true + default: benchmark + type: string jobs: run-e2e-tests-workflow: name: Run E2E Tests uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml with: - test_ids: 'benchmark/keeper_test.go:TestAutomationBenchmark' + test_ids: '${{ inputs.testType }}/automation_test.go:TestAutomationBenchmark' test_config_override_path: ${{ inputs.test_config_override_path }} SLACK_USER: ${{ inputs.slackMemberID }} SLACK_CHANNEL: C03KJ5S7KEK diff --git a/integration-tests/testconfig/automation/automation.toml b/integration-tests/testconfig/automation/automation.toml index f07136e998e..b2e87fa34f9 100644 --- a/integration-tests/testconfig/automation/automation.toml +++ b/integration-tests/testconfig/automation/automation.toml @@ -2,6 +2,9 @@ [Common] chainlink_node_funding = 2.0 +[Pyroscope] +enabled=false + [NodeConfig] BaseConfigTOML = """ [Feature] @@ -207,6 +210,10 @@ max_perform_data_size=5_000 max_revert_data_size=5_000 # load test specific overrides +[Load.Logging.Grafana] +base_url="https://grafana.ops.prod.cldev.sh" +dashboard_url="/d/a4899f53-f709-430a-aec2-24f32198dcc9/chainlink-automation-v2-load-test" + [Load.Seth] ephemeral_addresses_number = 100 root_key_funds_buffer = 1_000_000 @@ -301,6 +308,9 @@ max_revert_data_size=5_000 enabled=false # automation benchmark test specific overrides +[Benchmark.Logging.Grafana] +base_url="https://grafana.ops.prod.cldev.sh" +dashboard_url="/d/Q8n6m1unz/chainlink-automation-benchmark-test" # will retry roughly for 1h before giving up (900 * 4s) [Benchmark.Automation.Resiliency] @@ -420,6 +430,9 @@ max_perform_data_size=5_000 max_revert_data_size=5_000 # automation soak test specific overrides +[Soak.Logging.Grafana] +base_url="https://grafana.ops.prod.cldev.sh" +dashboard_url="/d/Q8n6m1unz/chainlink-automation-benchmark-test" # will retry roughly for 1h before giving up (900 * 4s) [Soak.Automation.Resiliency] diff --git a/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml new file mode 100644 index 00000000000..0b704524e9d --- /dev/null +++ b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_1.toml @@ -0,0 +1,15 @@ +[ChainlinkImage] +version="latest" + +[Benchmark.Automation.Benchmark] +registry_to_test = "2_1" +number_of_registries = 1 +number_of_nodes = 6 +number_of_upkeeps = 1000 +upkeep_gas_limit = 1500000 +check_gas_to_burn = 10000 +perform_gas_to_burn = 1000 +block_range = 3600 +block_interval = 60 +forces_single_tx_key = false +delete_jobs_on_end = true \ No newline at end of file diff --git a/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml new file mode 100644 index 00000000000..f00bb5ed47b --- /dev/null +++ b/integration-tests/testconfig/automation/overrides/benchmark/1000Upkeeps-1h-2_3.toml @@ -0,0 +1,15 @@ +[ChainlinkImage] +version="latest" + +[Benchmark.Automation.Benchmark] +registry_to_test = "2_3" +number_of_registries = 1 +number_of_nodes = 6 +number_of_upkeeps = 1000 +upkeep_gas_limit = 1500000 +check_gas_to_burn = 10000 +perform_gas_to_burn = 1000 +block_range = 3600 +block_interval = 60 +forces_single_tx_key = false +delete_jobs_on_end = true \ No newline at end of file diff --git a/integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml b/integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml new file mode 100644 index 00000000000..6d58253f526 --- /dev/null +++ b/integration-tests/testconfig/automation/overrides/load/500Upkeeps-1x-1h.toml @@ -0,0 +1,47 @@ +[ChainlinkImage] +version="latest" + +[Load.Seth] +root_key_funds_buffer = 1_000_000 +ephemeral_addresses_number = 300 + +[Load.Seth.nonce_manager] +key_sync_timeout = "100s" + +[Load.Common] +chainlink_node_funding = 1000 + +[Load.Automation.AutomationConfig] +use_log_buffer_v1=false + +[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig] +block_rate=1 +log_limit=2 + +[Load.Automation] +[Load.Automation.General] +number_of_nodes=6 +duration=3600 +block_time=1 +spec_type="recommended" +chainlink_node_log_level="debug" +use_prometheus=true +remove_namespace = true + +[Load.Automation.DataStreams] +enabled=false + +[[Load.Automation.Load]] +number_of_upkeeps=500 +number_of_events = 1 +number_of_spam_matching_events = 0 +number_of_spam_non_matching_events = 0 +check_burn_amount = 0 +perform_burn_amount = 0 +upkeep_gas_limit = 1000000 +shared_trigger = false +is_streams_lookup = false +feeds = [] + +[Pyroscope] +enabled=false \ No newline at end of file diff --git a/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml new file mode 100644 index 00000000000..8991a6eb903 --- /dev/null +++ b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-12h.toml @@ -0,0 +1,43 @@ +[ChainlinkImage] +version="latest" + +[Load.Seth] +root_key_funds_buffer = 1_000_000 + +[Load.Common] +chainlink_node_funding = 10000 + +[Load.Automation.AutomationConfig] +use_log_buffer_v1=false + +[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig] +block_rate=1 +log_limit=2 + +[Load.Automation] +[Load.Automation.General] +number_of_nodes=6 +duration=43200 +block_time=1 +spec_type="recommended" +chainlink_node_log_level="debug" +use_prometheus=true +remove_namespace = true + +[Load.Automation.DataStreams] +enabled=false + +[[Load.Automation.Load]] +number_of_upkeeps=50 +number_of_events = 1 +number_of_spam_matching_events = 0 +number_of_spam_non_matching_events = 0 +check_burn_amount = 0 +perform_burn_amount = 0 +upkeep_gas_limit = 1000000 +shared_trigger = false +is_streams_lookup = false +feeds = [] + +[Pyroscope] +enabled=false \ No newline at end of file diff --git a/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml new file mode 100644 index 00000000000..a133fce6dcf --- /dev/null +++ b/integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1h.toml @@ -0,0 +1,43 @@ +[ChainlinkImage] +version="latest" + +[Load.Seth] +root_key_funds_buffer = 1_000_000 + +[Load.Common] +chainlink_node_funding = 1000 + +[Load.Automation.AutomationConfig] +use_log_buffer_v1=false + +[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig] +block_rate=1 +log_limit=2 + +[Load.Automation] +[Load.Automation.General] +number_of_nodes=6 +duration=3600 +block_time=1 +spec_type="recommended" +chainlink_node_log_level="debug" +use_prometheus=true +remove_namespace = true + +[Load.Automation.DataStreams] +enabled=false + +[[Load.Automation.Load]] +number_of_upkeeps=50 +number_of_events = 1 +number_of_spam_matching_events = 0 +number_of_spam_non_matching_events = 0 +check_burn_amount = 0 +perform_burn_amount = 0 +upkeep_gas_limit = 1000000 +shared_trigger = false +is_streams_lookup = false +feeds = [] + +[Pyroscope] +enabled=false \ No newline at end of file diff --git a/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml new file mode 100644 index 00000000000..fda5cb6ea29 --- /dev/null +++ b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_1.toml @@ -0,0 +1,15 @@ +[ChainlinkImage] +version="latest" + +[Soak.Automation.Benchmark] +registry_to_test = "2_1" +number_of_registries = 1 +number_of_nodes = 6 +number_of_upkeeps = 50 +upkeep_gas_limit = 1500000 +check_gas_to_burn = 10000 +perform_gas_to_burn = 1000 +block_range = 28800 +block_interval = 300 +forces_single_tx_key = false +delete_jobs_on_end = true \ No newline at end of file diff --git a/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml new file mode 100644 index 00000000000..46ba1ad3e85 --- /dev/null +++ b/integration-tests/testconfig/automation/overrides/soak/50Upkeeps-8h-2_3.toml @@ -0,0 +1,15 @@ +[ChainlinkImage] +version="latest" + +[Soak.Automation.Benchmark] +registry_to_test = "2_3" +number_of_registries = 1 +number_of_nodes = 6 +number_of_upkeeps = 50 +upkeep_gas_limit = 1500000 +check_gas_to_burn = 10000 +perform_gas_to_burn = 1000 +block_range = 28800 +block_interval = 300 +forces_single_tx_key = false +delete_jobs_on_end = true \ No newline at end of file diff --git a/integration-tests/testconfig/keeper/keeper.toml b/integration-tests/testconfig/keeper/keeper.toml index b4f544165a9..d483d6df493 100644 --- a/integration-tests/testconfig/keeper/keeper.toml +++ b/integration-tests/testconfig/keeper/keeper.toml @@ -32,4 +32,4 @@ HTTPSPort = 0 [Keeper] TurnLookBack = 0 -""" +""" \ No newline at end of file