CCIP-3428 Enabling ccip smoke test for testnet (#14452)

* enabling test for testnet

* home chain selector logic

* testnet deployment

* remove unwanted

* lint fix

* add readme

* fix typo

* fix

* make timer rely on test deadline

* update comment

* update comment

integration-tests/actions/actions.go

@@ -25,6 +25,7 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/lib/logging"
 	"github.com/smartcontractkit/chainlink-testing-framework/lib/testreporters"
 	"github.com/smartcontractkit/chainlink-testing-framework/lib/utils/conversions"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	ethContracts "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum"
 	"github.com/smartcontractkit/chainlink/integration-tests/wrappers"
@@ -285,7 +286,7 @@ func fundChainlinkNodesAtAnyKey(
 			return err
 		}
 
-		fromAddress, err := privateKeyToAddress(privateKey)
+		fromAddress, err := PrivateKeyToAddress(privateKey)
 		if err != nil {
 			return err
 		}
@@ -336,7 +337,7 @@ type FundsToSendPayload struct {
 // to given address. You can override any or none of the following: gas limit, gas price, gas fee cap, gas tip cap.
 // Values that are not set will be estimated or taken from config.
 func SendFunds(logger zerolog.Logger, client *seth.Client, payload FundsToSendPayload) (*types.Receipt, error) {
-	fromAddress, err := privateKeyToAddress(payload.PrivateKey)
+	fromAddress, err := PrivateKeyToAddress(payload.PrivateKey)
 	if err != nil {
 		return nil, err
 	}
@@ -910,7 +911,7 @@ func deployAnyOCRv1Contracts(
 	return ocrInstances, nil
 }
 
-func privateKeyToAddress(privateKey *ecdsa.PrivateKey) (common.Address, error) {
+func PrivateKeyToAddress(privateKey *ecdsa.PrivateKey) (common.Address, error) {
 	publicKey := privateKey.Public()
 	publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey)
 	if !ok {

integration-tests/actions/refund.go

@@ -234,6 +234,14 @@ func (r *OvershotTransferRetrier) Retry(ctx context.Context, logger zerolog.Logg
 // of strategies to attempt to return funds, including retrying with less funds if the transaction fails due to
 // insufficient funds, and retrying with a higher gas limit if the transaction fails due to gas too low.
 func ReturnFundsFromNodes(log zerolog.Logger, client *seth.Client, chainlinkNodes []contracts.ChainlinkNodeWithKeysAndAddress) error {
+	var keyExporters []contracts.ChainlinkKeyExporter
+	for _, node := range chainlinkNodes {
+		keyExporters = append(keyExporters, node)
+	}
+	return ReturnFundsFromKeyExporterNodes(log, client, keyExporters)
+}
+
+func ReturnFundsFromKeyExporterNodes(log zerolog.Logger, client *seth.Client, chainlinkNodes []contracts.ChainlinkKeyExporter) error {
 	if client == nil {
 		return fmt.Errorf("seth client is nil, unable to return funds from chainlink nodes")
 	}

integration-tests/ccip-tests/testsetups/test_env.go

@@ -332,6 +332,9 @@ func DeployLocalCluster(
 	// a func to start the CL nodes asynchronously
 	deployCL := func() error {
 		noOfNodes := pointer.GetInt(testInputs.EnvInput.NewCLCluster.NoOfNodes)
+		if env.ClCluster == nil {
+			env.ClCluster = &test_env.ClCluster{}
+		}
 		// if individual nodes are specified, then deploy them with specified configs
 		if len(testInputs.EnvInput.NewCLCluster.Nodes) > 0 {
 			for _, clNode := range testInputs.EnvInput.NewCLCluster.Nodes {

integration-tests/contracts/contract_models.go

@@ -151,9 +151,13 @@ type OffchainAggregatorData struct {
 type ChainlinkNodeWithKeysAndAddress interface {
 	MustReadOCRKeys() (*client.OCRKeys, error)
 	MustReadP2PKeys() (*client.P2PKeys, error)
-	ExportEVMKeysForChain(string) ([]*client.ExportedEVMKey, error)
 	PrimaryEthAddress() (string, error)
 	EthAddresses() ([]string, error)
+	ChainlinkKeyExporter
+}
+
+type ChainlinkKeyExporter interface {
+	ExportEVMKeysForChain(string) ([]*client.ExportedEVMKey, error)
 }
 
 type ChainlinkNodeWithForwarder interface {

integration-tests/deployment/README.md

@@ -19,11 +19,11 @@ environments like testnet/mainnet.
 - In-memory environment for fast integration testing
 - EVM only
 
-/deployment/docker
+/deployment/devenv
 - Coming soon
-- package name `docker`
+- package name `devenv`
 - Docker environment for higher fidelity testing
-- Support non-EVMs
+- Support non-EVMs (yet to be implemented)
 
 /deployment/ccip
 - package name `ccipdeployment`

integration-tests/deployment/ccip/add_chain_test.go

@@ -154,8 +154,9 @@ func TestAddChainInbound(t *testing.T) {
 
 	// TODO: Send via all inbound lanes and use parallel helper
 	// Now that the proposal has been executed we expect to be able to send traffic to this new 4th chain.
-	startBlock, err := e.Env.Chains[newChain].LatestBlockNum(testcontext.Get(t))
+	latesthdr, err := e.Env.Chains[newChain].Client.HeaderByNumber(testcontext.Get(t), nil)
 	require.NoError(t, err)
+	startBlock := latesthdr.Number.Uint64()
 	seqNr := SendRequest(t, e.Env, state, initialDeploy[0], newChain, true)
 	require.NoError(t,
 		ConfirmExecWithSeqNr(t, e.Env.Chains[initialDeploy[0]], e.Env.Chains[newChain], state.Chains[newChain].OffRamp, &startBlock, seqNr))

integration-tests/deployment/ccip/add_lane_test.go

@@ -50,8 +50,9 @@ func TestAddLane(t *testing.T) {
 			require.Len(t, offRamps, 0)
 		}
 	}
-	startBlock, err := e.Env.Chains[to].LatestBlockNum(testcontext.Get(t))
+	latesthdr, err := e.Env.Chains[to].Client.HeaderByNumber(testcontext.Get(t), nil)
 	require.NoError(t, err)
+	startBlock := latesthdr.Number.Uint64()
 	seqNum := SendRequest(t, e.Env, state, from, to, false)
 	require.Equal(t, uint64(1), seqNum)
 	require.NoError(t, ConfirmExecWithSeqNr(t, e.Env.Chains[from], e.Env.Chains[to], state.Chains[to].OffRamp, &startBlock, seqNum))

integration-tests/deployment/ccip/changeset/2_initial_deploy_test.go

@@ -62,8 +62,9 @@ func Test0002_InitialDeploy(t *testing.T) {
 			if src == dest {
 				continue
 			}
-			block, err := destChain.LatestBlockNum(testcontext.Get(t))
+			latesthdr, err := destChain.Client.HeaderByNumber(testcontext.Get(t), nil)
 			require.NoError(t, err)
+			block := latesthdr.Number.Uint64()
 			startBlocks[dest] = &block
 			seqNum := ccipdeployment.SendRequest(t, e, state, src, dest, false)
 			expectedSeqNum[dest] = seqNum

integration-tests/deployment/ccip/test_assertions.go

@@ -78,7 +78,15 @@ func ConfirmCommitWithExpectedSeqNumRange(
 	}
 
 	defer subscription.Unsubscribe()
-	timer := time.NewTimer(5 * time.Minute)
+	var duration time.Duration
+	deadline, ok := t.Deadline()
+	if ok {
+		// make this timer end a minute before so that we don't hit the deadline
+		duration = deadline.Sub(time.Now().Add(-1 * time.Minute))
+	} else {
+		duration = 5 * time.Minute
+	}
+	timer := time.NewTimer(duration)
 	defer timer.Stop()
 	ticker := time.NewTicker(2 * time.Second)
 	defer ticker.Stop()
@@ -97,8 +105,8 @@ func ConfirmCommitWithExpectedSeqNumRange(
 		case subErr := <-subscription.Err():
 			return fmt.Errorf("subscription error: %w", subErr)
 		case <-timer.C:
-			return fmt.Errorf("timed out waiting for commit report on chain selector %d from source selector %d expected seq nr range %s",
-				dest.Selector, src.Selector, expectedSeqNumRange.String())
+			return fmt.Errorf("timed out after waiting %s duration for commit report on chain selector %d from source selector %d expected seq nr range %s",
+				duration.String(), dest.Selector, src.Selector, expectedSeqNumRange.String())
 		case report := <-sink:
 			if len(report.Report.MerkleRoots) > 0 {
 				// Check the interval of sequence numbers and make sure it matches

integration-tests/deployment/ccip/test_helpers.go

@@ -8,6 +8,8 @@ import (
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 
+	"github.com/smartcontractkit/chainlink-testing-framework/lib/logging"
+
 	"github.com/ethereum/go-ethereum/common"
 	chainsel "github.com/smartcontractkit/chain-selectors"
 	"github.com/stretchr/testify/require"
@@ -158,7 +160,9 @@ func SendRequest(t *testing.T, e deployment.Environment, state CCIPOnChainState,
 	}, []uint64{dest})
 	require.NoError(t, err)
 	require.True(t, it.Next())
-	return it.Event.Message.Header.SequenceNumber
+	seqNum := it.Event.Message.Header.SequenceNumber
+	t.Logf("CCIP message sent from chain selector %d to chain selector %d tx %s seqNum %d", src, dest, tx.Hash().String(), seqNum)
+	return seqNum
 }
 
 // DeployedLocalDevEnvironment is a helper struct for setting up a local dev environment with docker
@@ -179,15 +183,13 @@ func NewDeployedLocalDevEnvironment(t *testing.T, lggr logger.Logger) DeployedLo
 	require.NotEmpty(t, envConfig.JDConfig, "jdUrl should not be empty")
 	chains, err := devenv.NewChains(lggr, envConfig.Chains)
 	require.NoError(t, err)
-	homeChainSel := uint64(0)
-	homeChainEVM := uint64(0)
+	// locate the home chain
+	homeChainSel := envConfig.HomeChainSelector
+	require.NotEmpty(t, homeChainSel, "homeChainSel should not be empty")
+	homeChainEVM, err := chainsel.ChainIdFromSelector(homeChainSel)
+	require.NoError(t, err)
+	require.NotEmpty(t, homeChainEVM, "homeChainEVM should not be empty")
 
-	// Say first chain is home chain.
-	for chainSel := range chains {
-		homeChainEVM, _ = chainsel.ChainIdFromSelector(chainSel)
-		homeChainSel = chainSel
-		break
-	}
 	// deploy the capability registry
 	ab, capReg, err := DeployCapReg(lggr, chains, homeChainSel)
 	require.NoError(t, err)
@@ -205,10 +207,9 @@ func NewDeployedLocalDevEnvironment(t *testing.T, lggr logger.Logger) DeployedLo
 	require.NoError(t, err)
 	require.NotNil(t, e)
 	require.NotNil(t, don)
-
+	zeroLogLggr := logging.GetTestLogger(t)
 	// fund the nodes
-	require.NoError(t, don.FundNodes(ctx, deployment.E18Mult(10), e.Chains))
-
+	devenv.FundNodes(t, zeroLogLggr, testEnv, cfg, don.PluginNodes())
 	return DeployedLocalDevEnvironment{
 		Ab:           ab,
 		Env:          *e,

integration-tests/deployment/devenv/.sample.env

@@ -2,4 +2,24 @@ E2E_JD_IMAGE=<job-distributor-image>
 E2E_JD_VERSION=<job-distributor-version>
 
 E2E_TEST_CHAINLINK_IMAGE=public.ecr.aws/w0i8p0z9/chainlink-ccip
-E2E_TEST_CHAINLINK_VERSION=2.14.0-ccip1.5.0
+E2E_TEST_CHAINLINK_VERSION=2.14.0-ccip1.5.0
+
+# RPC Configuration
+E2E_TEST_SEPOLIA_WALLET_KEY=<sepolia-wallet-key>
+E2E_TEST_SEPOLIA_RPC_HTTP_URL_1=<sepolia-rpc-http-url>
+E2E_TEST_SEPOLIA_RPC_HTTP_URL_2=<sepolia-rpc-http-url>
+E2E_TEST_SEPOLIA_RPC_WS_URL_1=<rpc-ws-url>
+E2E_TEST_SEPOLIA_RPC_WS_URL_2=<rpc-ws-url>
+
+E2E_TEST_AVALANCHE_FUJI_WALLET_KEY=<avalanche-wallet-key>
+E2E_TEST_AVALANCHE_FUJI_RPC_HTTP_URL_1=<avalanche-rpc-http-url>
+E2E_TEST_AVALANCHE_FUJI_RPC_HTTP_URL_2=<avalanche-rpc-http-url>
+E2E_TEST_AVALANCHE_FUJI_RPC_WS_URL_1=<avalanche-rpc-ws-url>
+E2E_TEST_AVALANCHE_FUJI_RPC_WS_URL_2=<avalanche-rpc-ws-url>
+
+E2E_TEST_BSC_TESTNET_WALLET_KEY=<bsc-wallet-key>
+E2E_TEST_BSC_TESTNET_RPC_HTTP_URL_1=<bsc-rpc-http-url>
+E2E_TEST_BSC_TESTNET_RPC_HTTP_URL_2=<bsc-rpc-http-url>
+E2E_TEST_BSC_TESTNET_RPC_WS_URL_1=<bsc-rpc-ws-url>
+E2E_TEST_BSC_TESTNET_RPC_WS_URL_2=<bsc-rpc-ws-url>
+

integration-tests/deployment/devenv/README.md

@@ -0,0 +1,45 @@
+## Overview
+
+This package is used to create ephemeral environment for local/CI testing. 
+It sets up an environment with local Docker containers running Chainlink nodes and a job distributor. 
+It can either create new simulated private Ethereum network containers or connect to existing testnets/mainnets.
+
+### Run Tests with Devenv
+
+The tests created with this environment are run as [end-to-end integration smoke tests](../../smoke).
+
+#### Setting Up Testconfig with Simulated Private Ethereum Network
+
+To run tests (e.g., [ccip-test](../../smoke/ccip_test.go)), 
+you need to set up the testconfig following the [testconfig setup instructions](../../testconfig/README.md). 
+The testconfig specifies the details of the different configurations to set up the environment and run the tests. 
+Generally, tests are run with the [default](../../testconfig/default.toml) config unless overridden by product-specific config. 
+For example, the [ccip-test](../../smoke/ccip_test.go) uses [ccip.toml](../../testconfig/ccip/ccip.toml) to specify 
+CCIP-specific test environment details.
+
+There are additional secret configuration parameters required by the `devenv` environment that are not stored in the testconfig. 
+These are read from environment variables. For example, Chainlink image, Job-Distributor image, etc. 
+All such environment variables are listed in the [sample.env](./.sample.env) file. 
+You can create a `.env` file in the same directory of the test and set the required environment variables.
+
+#### Setting Up Testconfig for Running Tests with Existing Testnet/Mainnet
+
+To run tests with existing testnet/mainnet, set up the testconfig with the details of the testnet/mainnet networks. 
+Following the integration-test [testconfig framework](../../testconfig/README.md#configuration-and-overrides), 
+create a new `overrides.toml` file with testnet/mainnet network details and place it under any location in the `integration-tests` directory. 
+By default, tests are run with private Ethereum network containers set up in the same Docker network as 
+the Chainlink nodes and job distributor. To run tests against existing testnet/mainnet, 
+set the `selected_network` field in the testconfig with the specific network names.
+
+For example, if running [ccip-smoke](../../smoke/ccip_test.go) tests with Sepolia, Avax, and Binance testnets, 
+copy the contents of [sepolia_avax_binance.toml](../../testconfig/ccip/overrides/sepolia_avax_binance.toml) 
+to the `overrides.toml` file.
+
+Before running the test, ensure that RPC and wallet secrets are set as environment variables. 
+Refer to the environment variables pattern in the [sample.env](./.sample.env) file to 
+provide necessary secrets applicable to the network you are running the tests against:
+- `E2E_TEST_<networkName>_WALLET_KEY_<sequence_number>`
+- `E2E_TEST_<networkName>_RPC_HTTP_URL_<sequence_number>`
+- `E2E_TEST_<networkName>_RPC_WS_URL_<sequence_number>`
+
+Now you are all set to run the tests with the existing testnet/mainnet.