CCIPv2: RPCLatency test, Full chaos suite

deployment/environment/crib/ccip_deployer.go

@@ -213,7 +213,7 @@ func ConnectCCIPLanes(ctx context.Context, lggr logger.Logger, envConfig devenv.
 
 // ConfigureCCIPOCR is a group of changesets used from CRIB to redeploy the chainlink don on an existing setup
 func ConfigureCCIPOCR(ctx context.Context, lggr logger.Logger, envConfig devenv.EnvironmentConfig, homeChainSel, feedChainSel uint64, ab deployment.AddressBook) (DeployCCIPOutput, error) {
-	e, _, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig)
+	e, don, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig)
 	if err != nil {
 		return DeployCCIPOutput{}, fmt.Errorf("failed to initiate new environment: %w", err)
 	}
@@ -224,6 +224,10 @@ func ConfigureCCIPOCR(ctx context.Context, lggr logger.Logger, envConfig devenv.
 	if err != nil {
 		return DeployCCIPOutput{}, fmt.Errorf("failed to apply changesets for setting up OCR: %w", err)
 	}
+	err = distributeTransmitterFunds(lggr, don.PluginNodes(), *e)
+	if err != nil {
+		return DeployCCIPOutput{}, err
+	}
 
 	addresses, err := e.ExistingAddresses.Addresses()
 	if err != nil {

integration-tests/load/ccip/README.md

@@ -1,4 +1,4 @@
-# CCIP Load Tests 1.6+
+# CCIP Load and Chaos Tests 1.6+
 
 Resources
 - [CRIB setup](https://smartcontract-it.atlassian.net/wiki/spaces/CRIB/pages/1024622593/CCIP+v2+CRIB+-+Deploy+Access+Instructions+WIP)
@@ -43,5 +43,58 @@ export TIMEOUT=6h
 go test -run ^TestCCIPLoad_RPS$ ./integration-tests/load/ccip -v -timeout $TIMEOUT`
 ```
 
-### Remote
-Update the `PROVIDER=aws` and `DEVSPACE_NAMESPACE` in crib environment and deploy. Everything else should be the same. 
+## Remote
+Update the `PROVIDER=aws` and `DEVSPACE_NAMESPACE` in crib environment and deploy. Everything else should be the same. 
+
+## Running Chaos Tests (Remote only)
+
+### Realistic RPC Latency
+
+Go to `integration-tests/testconfig/ccip/ccip.toml` and change params as required, select chaos mode first
+```
+[Load.CCIP.Load]
+# 0 - no chaos, 1 - rpc latency, 2 - full chaos suite
+ChaosMode = 1
+
+# works only with Load.CCIP.ChaosMode = 0
+RPCLatency = "400ms"
+RPCJitter = "50ms"
+```
+Prefer using `ChaosMode = 1` with `400ms` RPC latency by default
+
+Run the load test
+```
+go test -run ^TestCCIPLoad_RPS$ -v -timeout 12h
+```
+
+### Full Chaos Suite
+
+Go to `integration-tests/testconfig/ccip/ccip.toml` and change params as required, select chaos mode first
+
+```
+[Load.CCIP.Load]
+# 0 - no chaos, 1 - rpc latency, 2 - full chaos suite
+ChaosMode = 2
+```
+
+Then check chaos settings
+```
+[CCIP.Chaos]
+# works only with Load.CCIP.ChaosMode = 2
+Namespace = "crib-ccip-chaos-tests"
+# RPC, commit, exec dashboards, can be found here: https://grafana.ops.prod.cldev.sh/d/dde396ff-5d22-42fb-9e92-00845c17688c/load-ccipv2-exec-plugin-v2?orgId=1&editview=dashboard_json
+DashboardUIDs = ["e08d9f98-a39a-4603-8b44-e9a2958330e4", "ed3d5742-57cb-440f-b432-65f229c124ec", "dde396ff-5d22-42fb-9e92-00845c17688c"]
+WaitBeforeStart = "30s"
+# Chaos experiment total duration (chaos + recovery)
+ExperimentFullInterval = "10m"
+# Chaos time
+ExperimentInjectionInterval = "5m"
+# Src and Dst chain URLs, change it according to your namespace name
+SrcChainURL = "https://crib-ccip-chaos-tests-geth-1337-http.main.stage.cldev.sh"
+DstChainURL = "https://crib-ccip-chaos-tests-geth-2337-http.main.stage.cldev.sh"
+```
+
+Run the load test
+```
+go test -run ^TestCCIPLoad_RPS$ -v -timeout 12h
+```