diff --git a/integration-tests/load/ccip/README.md b/integration-tests/load/ccip/README.md index 3754a08e60e..7f54becdc1d 100644 --- a/integration-tests/load/ccip/README.md +++ b/integration-tests/load/ccip/README.md @@ -44,4 +44,21 @@ go test -run ^TestCCIPLoad_RPS$ ./integration-tests/load/ccip -v -timeout $TIMEO ``` ### Remote -Update the `PROVIDER=aws` and `DEVSPACE_NAMESPACE` in crib environment and deploy. Everything else should be the same. \ No newline at end of file +Update the `PROVIDER=aws` and `DEVSPACE_NAMESPACE` in crib environment and deploy. Everything else should be the same. + +### Running Chaos Tests + +Go to `integration-tests/testconfig/ccip/ccip.toml` and change params as required +``` +[CCIP.Chaos] +Namespace = "crib-ccip-chaos" +ExperimentFullInterval = "3m" +ExperimentInjectionInterval = "90s" +SrcChainURL = "$your_crib_src_chain_url" +DstChainURL = "$your_crib_dst_chain_url" +``` + +Run your load test, then start the chaos suite +``` +go test -v -timeout 12h -run TestK8sChaos +``` \ No newline at end of file diff --git a/integration-tests/load/ccip/ccip_chaos_test.go b/integration-tests/load/ccip/ccip_chaos_test.go new file mode 100644 index 00000000000..cdfc53cdc6d --- /dev/null +++ b/integration-tests/load/ccip/ccip_chaos_test.go @@ -0,0 +1,239 @@ +package ccip + +import ( + "context" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink-testing-framework/framework/rpc" + "github.com/smartcontractkit/chainlink-testing-framework/havoc" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + "github.com/smartcontractkit/chainlink/load-tests/ccip/template" + "github.com/stretchr/testify/require" + "os" + "testing" + "time" +) + +func TestK8sChaos(t *testing.T) { + l := log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(zerolog.DebugLevel) + c, err := havoc.NewChaosMeshClient() + require.NoError(t, err) + + config, err := tc.GetConfig([]string{"Chaos"}, tc.CCIP) + require.NoError(t, err) + cfg := config.CCIP.Chaos + + testCases := []struct { + name string + run func(t *testing.T) + validate func(t *testing.T) + }{ + // pod failures + { + name: "Fail src chain", + run: func(t *testing.T) { + src, err := template.PodFail(c, l, template.PodFailCfg{ + Namespace: cfg.Namespace, + LabelKey: "instance", + LabelValues: []string{"geth-1337"}, + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + src.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Fail dst chain", + run: func(t *testing.T) { + dst, err := template.PodFail(c, l, template.PodFailCfg{ + Namespace: cfg.Namespace, + LabelKey: "instance", + LabelValues: []string{"geth-2337"}, + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + dst.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Fail one node", + run: func(t *testing.T) { + node1, err := template.PodFail(c, l, template.PodFailCfg{ + Namespace: cfg.Namespace, + LabelKey: "app.kubernetes.io/instance", + LabelValues: []string{"ccip-0"}, + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + node1.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Fail two nodes", + run: func(t *testing.T) { + node1, err := template.PodFail(c, l, template.PodFailCfg{ + Namespace: cfg.Namespace, + LabelKey: "app.kubernetes.io/instance", + LabelValues: []string{"ccip-0", "ccip-1"}, + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + node1.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + // network delay + { + name: "Slow src chain", + run: func(t *testing.T) { + src, err := template.PodDelay(c, l, template.PodDelayCfg{ + Namespace: cfg.Namespace, + LabelKey: "instance", + LabelValues: []string{"geth-1337"}, + Latency: 200 * time.Millisecond, + Jitter: 200 * time.Millisecond, + Correlation: "0", + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + src.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Slow dst chain", + run: func(t *testing.T) { + src, err := template.PodDelay(c, l, template.PodDelayCfg{ + Namespace: cfg.Namespace, + LabelKey: "instance", + LabelValues: []string{"geth-2337"}, + Latency: 200 * time.Millisecond, + Jitter: 200 * time.Millisecond, + Correlation: "0", + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + src.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "One slow node", + run: func(t *testing.T) { + src, err := template.PodDelay(c, l, template.PodDelayCfg{ + Namespace: cfg.Namespace, + LabelKey: "app.kubernetes.io/instance", + LabelValues: []string{"ccip-0"}, + Latency: 200 * time.Millisecond, + Jitter: 200 * time.Millisecond, + Correlation: "0", + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + src.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Two slow nodes", + run: func(t *testing.T) { + src, err := template.PodDelay(c, l, template.PodDelayCfg{ + Namespace: cfg.Namespace, + LabelKey: "app.kubernetes.io/instance", + LabelValues: []string{"ccip-0", "ccip-1"}, + Latency: 200 * time.Millisecond, + Jitter: 200 * time.Millisecond, + Correlation: "0", + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + src.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "One node partition", + run: func(t *testing.T) { + src, err := template.PodPartition(c, l, template.PodPartitionCfg{ + Namespace: cfg.Namespace, + LabelFromKey: "app.kubernetes.io/instance", + LabelFromValues: []string{"ccip-0"}, + LabelToKey: "app.kubernetes.io/instance", + LabelToValues: []string{"ccip-1", "ccip-2", "ccip-3"}, + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + src.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Two nodes partition", + run: func(t *testing.T) { + src, err := template.PodPartition(c, l, template.PodPartitionCfg{ + Namespace: cfg.Namespace, + LabelFromKey: "app.kubernetes.io/instance", + LabelFromValues: []string{"ccip-0", "ccip-1"}, + LabelToKey: "app.kubernetes.io/instance", + LabelToValues: []string{"ccip-2", "ccip-3"}, + InjectionDuration: cfg.GetExperimentInjectionInterval(), + }) + require.NoError(t, err) + src.Create(context.Background()) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Reorg src chain below finality", + run: func(t *testing.T) { + r := rpc.New(cfg.SrcChainURL, nil) + err := r.GethSetHead(cfg.ReorgDepthBelowFinality) + require.NoError(t, err) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Reorg dst chain below finality", + run: func(t *testing.T) { + r := rpc.New(cfg.DstChainURL, nil) + err := r.GethSetHead(cfg.ReorgDepthBelowFinality) + require.NoError(t, err) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Reorg src chain above finality", + run: func(t *testing.T) { + r := rpc.New(cfg.SrcChainURL, nil) + err := r.GethSetHead(cfg.ReorgDepthAboveFinality) + require.NoError(t, err) + }, + validate: func(t *testing.T) {}, + }, + { + name: "Reorg dst chain above finality", + run: func(t *testing.T) { + r := rpc.New(cfg.DstChainURL, nil) + err := r.GethSetHead(cfg.ReorgDepthAboveFinality) + require.NoError(t, err) + }, + validate: func(t *testing.T) {}, + }, + } + + // Start WASP load test here, apply average load profile that you expect in production! + // Configure timeouts and validate all the test cases until the test ends + // or start the load test manually + + // Run test cases + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + testCase.run(t) + time.Sleep(cfg.GetExperimentInterval()) + testCase.validate(t) + }) + } +} diff --git a/integration-tests/load/ccip/template/experiments.go b/integration-tests/load/ccip/template/experiments.go new file mode 100644 index 00000000000..1f3016c0fc0 --- /dev/null +++ b/integration-tests/load/ccip/template/experiments.go @@ -0,0 +1,258 @@ +package template + +import ( + "fmt" + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/google/uuid" + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/havoc" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +func defaultListeners(l zerolog.Logger) []havoc.ChaosListener { + return []havoc.ChaosListener{ + havoc.NewChaosLogger(l), + } +} + +type PodPartitionCfg struct { + Namespace string + Description string + LabelFromKey string + LabelFromValues []string + LabelToKey string + LabelToValues []string + InjectionDuration time.Duration + ExperimentCreateDelay time.Duration +} + +func PodPartition(client client.Client, l zerolog.Logger, cfg PodPartitionCfg) (*havoc.Chaos, error) { + return havoc.NewChaos(havoc.ChaosOpts{ + Object: &v1alpha1.NetworkChaos{ + TypeMeta: metav1.TypeMeta{ + Kind: string(v1alpha1.TypeNetworkChaos), + APIVersion: "chaos-mesh.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("partition-%s", uuid.NewString()[0:5]), + Namespace: cfg.Namespace, + }, + Spec: v1alpha1.NetworkChaosSpec{ + Action: v1alpha1.PartitionAction, + Duration: ptr.To[string]((cfg.InjectionDuration).String()), + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.Namespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Operator: "In", + Key: cfg.LabelFromKey, + Values: cfg.LabelFromValues, + }, + }, + }, + }, + }, + Target: &v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.Namespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Operator: "In", + Key: cfg.LabelToKey, + Values: cfg.LabelToValues, + }, + }, + }, + }, + }, + }, + }, + Listeners: defaultListeners(l), + Logger: &l, + Client: client, + }) +} + +type PodDelayCfg struct { + Namespace string + Description string + Latency time.Duration + Jitter time.Duration + Correlation string + LabelKey string + LabelValues []string + InjectionDuration time.Duration + ExperimentCreateDelay time.Duration +} + +func PodDelay(client client.Client, l zerolog.Logger, cfg PodDelayCfg) (*havoc.Chaos, error) { + return havoc.NewChaos(havoc.ChaosOpts{ + Object: &v1alpha1.NetworkChaos{ + TypeMeta: metav1.TypeMeta{ + Kind: string(v1alpha1.TypeNetworkChaos), + APIVersion: "chaos-mesh.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("delay-%s", uuid.NewString()[0:5]), + Namespace: cfg.Namespace, + }, + Spec: v1alpha1.NetworkChaosSpec{ + Action: v1alpha1.DelayAction, + Duration: ptr.To[string]((cfg.InjectionDuration).String()), + TcParameter: v1alpha1.TcParameter{ + Delay: &v1alpha1.DelaySpec{ + Latency: cfg.Latency.String(), + Correlation: cfg.Correlation, + Jitter: cfg.Jitter.String(), + }, + }, + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.Namespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Operator: "In", + Key: cfg.LabelKey, + Values: cfg.LabelValues, + }, + }, + }, + }, + }, + }, + }, + Listeners: defaultListeners(l), + Logger: &l, + Client: client, + }) +} + +type PodFailCfg struct { + Namespace string + Description string + LabelKey string + LabelValues []string + InjectionDuration time.Duration + ExperimentCreateDelay time.Duration +} + +func PodFail(client client.Client, l zerolog.Logger, cfg PodFailCfg) (*havoc.Chaos, error) { + return havoc.NewChaos(havoc.ChaosOpts{ + Description: cfg.Description, + DelayCreate: cfg.ExperimentCreateDelay, + Object: &v1alpha1.PodChaos{ + TypeMeta: metav1.TypeMeta{ + Kind: string(v1alpha1.TypePodChaos), + APIVersion: "chaos-mesh.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("fail-%s", uuid.NewString()[0:5]), + Namespace: cfg.Namespace, + }, + Spec: v1alpha1.PodChaosSpec{ + Action: v1alpha1.PodFailureAction, + Duration: ptr.To[string](cfg.InjectionDuration.String()), + ContainerSelector: v1alpha1.ContainerSelector{ + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.Namespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Operator: "In", + Key: cfg.LabelKey, + Values: cfg.LabelValues, + }, + }, + }, + }, + }, + }, + }, + }, + Listeners: defaultListeners(l), + Logger: &l, + Client: client, + }) +} + +type NodeCPUStressConfig struct { + Namespace string + Description string + Cores int + CoreLoadPercentage int // 0-100 + LabelKey string + LabelValues []string + InjectionDuration time.Duration + ExperimentTotalDuration time.Duration + ExperimentCreateDelay time.Duration +} + +func podStressCPU(client client.Client, l zerolog.Logger, cfg NodeCPUStressConfig) (*havoc.Schedule, error) { + return havoc.NewSchedule(havoc.ScheduleOpts{ + Description: cfg.Description, + DelayCreate: cfg.ExperimentCreateDelay, + Duration: cfg.ExperimentTotalDuration, + Logger: &l, + Object: &v1alpha1.Schedule{ + TypeMeta: metav1.TypeMeta{ + Kind: "Schedule", + APIVersion: "chaos-mesh.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "stress", + Namespace: cfg.Namespace, + }, + Spec: v1alpha1.ScheduleSpec{ + Schedule: "@every 1m", + ConcurrencyPolicy: v1alpha1.ForbidConcurrent, + Type: v1alpha1.ScheduleTypeStressChaos, + HistoryLimit: 2, + ScheduleItem: v1alpha1.ScheduleItem{ + EmbedChaos: v1alpha1.EmbedChaos{ + StressChaos: &v1alpha1.StressChaosSpec{ + ContainerSelector: v1alpha1.ContainerSelector{ + PodSelector: v1alpha1.PodSelector{ + Mode: v1alpha1.AllMode, + Selector: v1alpha1.PodSelectorSpec{ + GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ + Namespaces: []string{cfg.Namespace}, + ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ + { + Operator: "In", + Key: cfg.LabelKey, + Values: cfg.LabelValues, + }, + }, + }, + }, + }, + }, + Stressors: &v1alpha1.Stressors{ + CPUStressor: &v1alpha1.CPUStressor{ + Stressor: v1alpha1.Stressor{ + Workers: cfg.Cores, + }, + Load: ptr.To[int](cfg.CoreLoadPercentage), + }, + }, + Duration: ptr.To[string](cfg.InjectionDuration.String()), + }, + }, + }, + }, + }, + Client: client, + }) +} diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 59699afc53d..ed0abdb0231 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -21,8 +21,10 @@ require ( require ( github.com/K-Phoen/grabana v0.22.2 + github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a github.com/ethereum/go-ethereum v1.14.11 github.com/go-resty/resty/v2 v2.15.3 + github.com/google/uuid v1.6.0 github.com/pelletier/go-toml/v2 v2.2.3 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 @@ -31,6 +33,8 @@ require ( github.com/smartcontractkit/chainlink-ccip v0.0.0-20250207140936-540f8266d0c7 github.com/smartcontractkit/chainlink-common v0.4.2-0.20250130202959-6f1f48342e36 github.com/smartcontractkit/chainlink-integrations/evm v0.0.0-20250207143947-22151a16f100 + github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7 + github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 github.com/smartcontractkit/chainlink-testing-framework/lib v1.51.0 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 @@ -41,6 +45,9 @@ require ( go.uber.org/ratelimit v0.3.1 golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c golang.org/x/sync v0.10.0 + k8s.io/apimachinery v0.31.2 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + sigs.k8s.io/controller-runtime v0.19.0 ) require ( @@ -121,7 +128,6 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/cockroachdb/errors v1.11.3 // indirect @@ -239,7 +245,6 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect @@ -419,8 +424,6 @@ require ( github.com/smartcontractkit/chainlink-protos/orchestrator v0.4.0 // indirect github.com/smartcontractkit/chainlink-protos/svr v0.0.0-20250123084029-58cce9b32112 // indirect github.com/smartcontractkit/chainlink-solana v1.1.2-0.20250205221351-c3ca04743e06 // indirect - github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7 // indirect - github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 // indirect github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 // indirect @@ -528,17 +531,14 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.31.2 // indirect k8s.io/apiextensions-apiserver v0.31.0 // indirect - k8s.io/apimachinery v0.31.2 // indirect k8s.io/cli-runtime v0.31.2 // indirect k8s.io/client-go v0.31.2 // indirect k8s.io/component-base v0.31.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f // indirect k8s.io/kubectl v0.31.2 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect pgregory.net/rapid v1.1.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect - sigs.k8s.io/controller-runtime v0.19.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.17.2 // indirect sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index 5032297da32..c494c11f61a 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -250,5 +250,12 @@ LoadDuration = "30s" # destination chain selectors to send messages to NumDestinationChains = 2 # Directory where we receive environment configuration from crib -CribEnvDirectory = "../../../../crib/deployments/ccip-v2/.tmp" -TimeoutDuration = "1h" \ No newline at end of file +CribEnvDirectory = "/Users/fahrenheit/GolandProjects/chainlink/integration-tests/load/ccip/crib-ccip-v2-load-test-data" +TimeoutDuration = "30m" + +[CCIP.Chaos] +Namespace = "default" +ExperimentFullInterval = "1m" +ExperimentInjectionInterval = "30s" +SrcChainURL = "http://localhost:8545" +DstChainURL = "http://localhost:8545" diff --git a/integration-tests/testconfig/ccip/chaos.go b/integration-tests/testconfig/ccip/chaos.go new file mode 100644 index 00000000000..2733871c6f2 --- /dev/null +++ b/integration-tests/testconfig/ccip/chaos.go @@ -0,0 +1,40 @@ +package ccip + +import ( + "testing" + "time" + + "github.com/smartcontractkit/chainlink/deployment" + + "github.com/stretchr/testify/require" +) + +type ChaosConfig struct { + Namespace string + ExperimentFullInterval string + ExperimentInjectionInterval string + ReorgDepthBelowFinality int + ReorgDepthAboveFinality int + SrcChainURL string + DstChainURL string +} + +func (l *ChaosConfig) Validate(t *testing.T, e *deployment.Environment) { + require.NotEmpty(t, l.Namespace, "k8s namespace can't be empty") + require.NotEmpty(t, l.ExperimentFullInterval, "experiment full interval can't be null, use Go time format 1h2m3s") + require.NotEmpty(t, l.ExperimentInjectionInterval, "experiment injection interval can't be null, use Go time format 1h2m3s") + require.NotEmpty(t, l.ReorgDepthBelowFinality, "reorg depth below finality can't be 0") + require.NotEmpty(t, l.ReorgDepthAboveFinality, "reorg depth above finality can't be 0") + require.NotEmpty(t, l.SrcChainURL, "src chain URL can't be null") + require.NotEmpty(t, l.DstChainURL, "src chain URL can't be null") +} + +func (l *ChaosConfig) GetExperimentInterval() time.Duration { + ld, _ := time.ParseDuration(l.ExperimentFullInterval) + return ld +} + +func (l *ChaosConfig) GetExperimentInjectionInterval() time.Duration { + ld, _ := time.ParseDuration(l.ExperimentInjectionInterval) + return ld +} diff --git a/integration-tests/testconfig/ccip/config.go b/integration-tests/testconfig/ccip/config.go index 374322041ec..74cd44ab1b0 100644 --- a/integration-tests/testconfig/ccip/config.go +++ b/integration-tests/testconfig/ccip/config.go @@ -39,6 +39,7 @@ type Config struct { FeedChainSelector *string `toml:",omitempty"` RMNConfig RMNConfig `toml:",omitempty"` Load *LoadConfig `toml:",omitempty"` + Chaos *ChaosConfig `toml:",omitempty"` } type RMNConfig struct {