From 011477216a9ef9e8aae74598f5f8254554c4682c Mon Sep 17 00:00:00 2001 From: "Simon B.Robert" Date: Thu, 13 Feb 2025 10:02:13 -0500 Subject: [PATCH 1/2] Add new RMN smoke test --- .github/e2e-tests.yml | 32 ++++++ integration-tests/smoke/ccip/ccip_rmn_test.go | 103 +++++++++++++++++- 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 2207ae4a38f..a674bc9c4f4 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -1111,6 +1111,38 @@ runner-test-matrix: E2E_RMN_AFN2PROXY_VERSION: 678509b CCIP_V16_TEST_ENV: docker + - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_IncorrectSig$ + path: integration-tests/smoke/ccip/ccip_rmn_test.go + test_env_type: docker + runs_on: ubuntu20.04-8cores-32GB + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_IncorrectSig$ -timeout 12m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_JD_VERSION: 0.6.0 + E2E_RMN_RAGEPROXY_VERSION: 678509b + E2E_RMN_AFN2PROXY_VERSION: 678509b + CCIP_V16_TEST_ENV: docker + + - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_TwoMessagesOnTwoLanesIncludingBatchingWithTemporaryPause$ + path: integration-tests/smoke/ccip/ccip_rmn_test.go + test_env_type: docker + runs_on: ubuntu20.04-8cores-32GB + triggers: + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/smoke/ccip && go test -test.run ^TestRMN_TwoMessagesOnTwoLanesIncludingBatchingWithTemporaryPause$ -timeout 12m -test.parallel=1 -count=1 -json + pyroscope_env: ci-smoke-ccipv1_6-evm-simulated + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + E2E_JD_VERSION: 0.6.0 + E2E_RMN_RAGEPROXY_VERSION: 678509b + E2E_RMN_AFN2PROXY_VERSION: 678509b + CCIP_V16_TEST_ENV: docker + # END: CCIPv1.6 tests # START: CCIP tests diff --git a/integration-tests/smoke/ccip/ccip_rmn_test.go b/integration-tests/smoke/ccip/ccip_rmn_test.go index 133152d9419..a0298865a48 100644 --- a/integration-tests/smoke/ccip/ccip_rmn_test.go +++ b/integration-tests/smoke/ccip/ccip_rmn_test.go @@ -2,6 +2,7 @@ package ccip import ( "context" + "crypto/ecdsa" "errors" "math/big" "slices" @@ -13,6 +14,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/rs/zerolog" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -32,6 +34,30 @@ import ( testsetups "github.com/smartcontractkit/chainlink/integration-tests/testsetups/ccip" ) +func TestRMN_IncorrectSig(t *testing.T) { + runRmnTestCase(t, rmnTestCase{ + nodesWithIncorrectSigner: []int{0, 1}, + name: "messages with incorrect RMN signature", + waitForExec: true, + passIfNoCommitAfter: 30 * time.Second, + homeChainConfig: homeChainConfig{ + f: map[int]int{chain0: 1, chain1: 1}, + }, + remoteChainsConfig: []remoteChainConfig{ + {chainIdx: chain0, f: 1}, + {chainIdx: chain1, f: 1}, + }, + rmnNodes: []rmnNode{ + {id: 0, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + {id: 1, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + {id: 2, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + }, + messagesToSend: []messageToSend{ + {fromChainIdx: chain0, toChainIdx: chain1, count: 1}, + }, + }) +} + func TestRMN_TwoMessagesOnTwoLanesIncludingBatching(t *testing.T) { runRmnTestCase(t, rmnTestCase{ name: "messages on two lanes including batching one lane RMN-enabled the other RMN-disabled", @@ -58,6 +84,29 @@ func TestRMN_TwoMessagesOnTwoLanesIncludingBatching(t *testing.T) { }) } +func TestRMN_TwoMessagesOnTwoLanesIncludingBatchingWithTemporaryPause(t *testing.T) { + runRmnTestCase(t, rmnTestCase{ + name: "messages on two lanes including batching", + waitForExec: true, + homeChainConfig: homeChainConfig{ + f: map[int]int{chain0: 1, chain1: 1}, + }, + remoteChainsConfig: []remoteChainConfig{ + {chainIdx: chain0, f: 1}, + {chainIdx: chain1, f: 1}, + }, + rmnNodes: []rmnNode{ + {id: 0, isSigner: true, observedChainIdxs: []int{chain0, chain1}, forceExit: true, restart: true}, + {id: 1, isSigner: true, observedChainIdxs: []int{chain0, chain1}, forceExit: true, restart: true}, + {id: 2, isSigner: true, observedChainIdxs: []int{chain0, chain1}}, + }, + messagesToSend: []messageToSend{ + {fromChainIdx: chain0, toChainIdx: chain1, count: 1}, + {fromChainIdx: chain1, toChainIdx: chain0, count: 5}, + }, + }) +} + func TestRMN_MultipleMessagesOnOneLaneNoWaitForExec(t *testing.T) { runRmnTestCase(t, rmnTestCase{ name: "multiple messages for rmn batching inspection and one rmn node down", @@ -309,7 +358,7 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { } rmnRemoteConfig[selector] = changeset.RMNRemoteConfig{ F: uint64(remoteCfg.f), - Signers: tc.pf.rmnRemoteSigners, + Signers: tc.alterSigners(t, tc.pf.rmnRemoteSigners), } } @@ -334,6 +383,9 @@ func runRmnTestCase(t *testing.T, tc rmnTestCase) { startBlocks, seqNumCommit, seqNumExec := tc.sendMessages(t, onChainState, envWithRMN) t.Logf("Sent all messages, seqNumCommit: %v seqNumExec: %v", seqNumCommit, seqNumExec) + cleanup := tc.restartNode(t, rmnCluster) + defer cleanup() + eg := errgroup.Group{} tc.callContractsToCurseChains(ctx, t, onChainState, envWithRMN) tc.callContractsToCurseAndRevokeCurse(ctx, &eg, t, onChainState, envWithRMN) @@ -437,6 +489,7 @@ type rmnNode struct { isSigner bool observedChainIdxs []int forceExit bool // force exit will simply force exit the rmn node to simulate failure scenarios + restart bool // restart will restart the rmn node to simulate failure scenarios } type messageToSend struct { @@ -458,6 +511,7 @@ type rmnTestCase struct { remoteChainsConfig []remoteChainConfig rmnNodes []rmnNode messagesToSend []messageToSend + nodesWithIncorrectSigner []int // populated fields after environment setup pf testCasePopulatedFields @@ -472,6 +526,29 @@ type testCasePopulatedFields struct { revokedCursedSubjectsPerChainSel map[uint64]map[uint64]time.Duration } +func (tc *rmnTestCase) alterSigners(t *testing.T, signers []rmn_remote.RMNRemoteSigner) []rmn_remote.RMNRemoteSigner { + for _, n := range tc.nodesWithIncorrectSigner { + for i, s := range signers { + if s.NodeIndex == uint64(n) { + // Random address ethereum private key + privateKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("failed to generate private key: %v", err) + } + publicKey := privateKey.Public() + publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey) + if !ok { + t.Fatalf("failed to cast public key to ECDSA") + } + address := crypto.PubkeyToAddress(*publicKeyECDSA) + signers[i].OnchainPublicKey = address + } + } + } + + return signers +} + func (tc *rmnTestCase) populateFields(t *testing.T, envWithRMN testhelpers.DeployedEnv, rmnCluster devenv.RMNCluster) { require.GreaterOrEqual(t, len(envWithRMN.Env.Chains), 2, "test assumes at least two chains") for _, chain := range envWithRMN.Env.Chains { @@ -561,6 +638,30 @@ func (tc rmnTestCase) killMarkedRmnNodes(t *testing.T, rmnCluster devenv.RMNClus } } +func (tc rmnTestCase) restartNode(t *testing.T, rmnCluster devenv.RMNCluster) func() { + go func() { + time.Sleep(10 * time.Second) + for _, n := range tc.rmnNodes { + if n.restart { + t.Logf("Restarting RMN node %d", n.id) + rmnN := rmnCluster.Nodes["rmn_"+strconv.Itoa(n.id)] + require.NoError(t, osutil.ExecCmd(zerolog.Nop(), "docker start "+rmnN.Proxy.ContainerName)) + t.Logf("Restarted RMN node %d", n.id) + } + } + }() + return func() { + for _, n := range tc.rmnNodes { + if n.restart { + t.Logf("Stopping RMN node %d", n.id) + rmnN := rmnCluster.Nodes["rmn_"+strconv.Itoa(n.id)] + require.NoError(t, osutil.ExecCmd(zerolog.Nop(), "docker stop "+rmnN.Proxy.ContainerName)) + t.Logf("Stopped RMN node %d", n.id) + } + } + } +} + func (tc rmnTestCase) disableOraclesIfThisIsACursingTestCase(ctx context.Context, t *testing.T, envWithRMN testhelpers.DeployedEnv) []string { disabledNodes := make([]string, 0) From 2dc385d3e41764dcc002fab7aa576d0ec015d00e Mon Sep 17 00:00:00 2001 From: "Simon B.Robert" Date: Thu, 13 Feb 2025 10:21:56 -0500 Subject: [PATCH 2/2] Fix linting issues --- integration-tests/smoke/ccip/ccip_rmn_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/integration-tests/smoke/ccip/ccip_rmn_test.go b/integration-tests/smoke/ccip/ccip_rmn_test.go index a0298865a48..a94fcb170fa 100644 --- a/integration-tests/smoke/ccip/ccip_rmn_test.go +++ b/integration-tests/smoke/ccip/ccip_rmn_test.go @@ -529,7 +529,7 @@ type testCasePopulatedFields struct { func (tc *rmnTestCase) alterSigners(t *testing.T, signers []rmn_remote.RMNRemoteSigner) []rmn_remote.RMNRemoteSigner { for _, n := range tc.nodesWithIncorrectSigner { for i, s := range signers { - if s.NodeIndex == uint64(n) { + if n >= 0 && s.NodeIndex == uint64(n) { // Random address ethereum private key privateKey, err := crypto.GenerateKey() if err != nil { @@ -639,17 +639,23 @@ func (tc rmnTestCase) killMarkedRmnNodes(t *testing.T, rmnCluster devenv.RMNClus } func (tc rmnTestCase) restartNode(t *testing.T, rmnCluster devenv.RMNCluster) func() { + errCh := make(chan error, 1) go func() { time.Sleep(10 * time.Second) for _, n := range tc.rmnNodes { if n.restart { t.Logf("Restarting RMN node %d", n.id) rmnN := rmnCluster.Nodes["rmn_"+strconv.Itoa(n.id)] - require.NoError(t, osutil.ExecCmd(zerolog.Nop(), "docker start "+rmnN.Proxy.ContainerName)) + if err := osutil.ExecCmd(zerolog.Nop(), "docker start "+rmnN.Proxy.ContainerName); err != nil { + errCh <- err + return + } t.Logf("Restarted RMN node %d", n.id) } } + errCh <- nil }() + require.NoError(t, <-errCh) return func() { for _, n := range tc.rmnNodes { if n.restart {