diff --git a/go.md b/go.md index 73361519bee..31c97886834 100644 --- a/go.md +++ b/go.md @@ -173,6 +173,8 @@ flowchart LR click chainlink-testing-framework/lib href "https://github.com/smartcontractkit/chainlink-testing-framework" chainlink-testing-framework/lib/grafana click chainlink-testing-framework/lib/grafana href "https://github.com/smartcontractkit/chainlink-testing-framework" + chainlink-testing-framework/sentinel --> chainlink-testing-framework/lib + click chainlink-testing-framework/sentinel href "https://github.com/smartcontractkit/chainlink-testing-framework" chainlink-testing-framework/seth click chainlink-testing-framework/seth href "https://github.com/smartcontractkit/chainlink-testing-framework" chainlink-testing-framework/wasp --> chainlink-testing-framework/lib/grafana @@ -186,6 +188,7 @@ flowchart LR chainlink/deployment --> mcms click chainlink/deployment href "https://github.com/smartcontractkit/chainlink" chainlink/integration-tests --> chainlink-testing-framework/havoc + chainlink/integration-tests --> chainlink-testing-framework/sentinel chainlink/integration-tests --> chainlink-testing-framework/wasp chainlink/integration-tests --> chainlink/deployment click chainlink/integration-tests href "https://github.com/smartcontractkit/chainlink" @@ -251,6 +254,7 @@ flowchart LR chainlink-testing-framework/havoc chainlink-testing-framework/lib chainlink-testing-framework/lib/grafana + chainlink-testing-framework/sentinel chainlink-testing-framework/seth chainlink-testing-framework/wasp end diff --git a/integration-tests/ccip-tests/actions/ccip_helpers.go b/integration-tests/ccip-tests/actions/ccip_helpers.go index ef429e8c034..3932fcfdddf 100644 --- a/integration-tests/ccip-tests/actions/ccip_helpers.go +++ b/integration-tests/ccip-tests/actions/ccip_helpers.go @@ -32,6 +32,8 @@ import ( "golang.org/x/exp/rand" "golang.org/x/sync/errgroup" + "github.com/smartcontractkit/chainlink-testing-framework/sentinel" + chainselectors "github.com/smartcontractkit/chain-selectors" commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" @@ -3571,38 +3573,7 @@ func (lane *CCIPLane) StartEventWatchers() error { for { select { case e := <-sendReqEventLatest: - lane.Logger.Info().Msgf("CCIPSendRequested event received for seq number %d", e.Message.SequenceNumber) - eventsForTx, ok := lane.Source.CCIPSendRequestedWatcher.Load(e.Raw.TxHash.Hex()) - if ok { - lane.Source.CCIPSendRequestedWatcher.Store(e.Raw.TxHash.Hex(), append(eventsForTx.([]*contracts.SendReqEventData), - &contracts.SendReqEventData{ - MessageId: e.Message.MessageId, - SequenceNumber: e.Message.SequenceNumber, - DataLength: len(e.Message.Data), - NoOfTokens: len(e.Message.TokenAmounts), - LogInfo: contracts.LogInfo{ - BlockNumber: e.Raw.BlockNumber, - TxHash: e.Raw.TxHash, - }, - Fee: e.Message.FeeTokenAmount, - })) - } else { - lane.Source.CCIPSendRequestedWatcher.Store(e.Raw.TxHash.Hex(), []*contracts.SendReqEventData{ - { - MessageId: e.Message.MessageId, - SequenceNumber: e.Message.SequenceNumber, - DataLength: len(e.Message.Data), - NoOfTokens: len(e.Message.TokenAmounts), - LogInfo: contracts.LogInfo{ - BlockNumber: e.Raw.BlockNumber, - TxHash: e.Raw.TxHash, - }, - Fee: e.Message.FeeTokenAmount, - }, - }) - } - - lane.Source.CCIPSendRequestedWatcher = testutils.DeleteNilEntriesFromMap(lane.Source.CCIPSendRequestedWatcher) + processSendRequestedEvent(lane, e) case <-lane.Context.Done(): return } @@ -3625,19 +3596,7 @@ func (lane *CCIPLane) StartEventWatchers() error { for { select { case e := <-reportAcceptedEvent: - lane.Logger.Info().Interface("Interval", e.Report.Interval).Msgf("ReportAccepted event received") - for i := e.Report.Interval.Min; i <= e.Report.Interval.Max; i++ { - lane.Dest.ReportAcceptedWatcher.Store(i, &contracts.CommitStoreReportAccepted{ - Min: e.Report.Interval.Min, - Max: e.Report.Interval.Max, - MerkleRoot: e.Report.MerkleRoot, - LogInfo: contracts.LogInfo{ - BlockNumber: e.Raw.BlockNumber, - TxHash: e.Raw.TxHash, - }, - }) - } - lane.Dest.ReportAcceptedWatcher = testutils.DeleteNilEntriesFromMap(lane.Dest.ReportAcceptedWatcher) + processReportAcceptedEvent(lane, e) case <-lane.Context.Done(): return } @@ -3660,13 +3619,7 @@ func (lane *CCIPLane) StartEventWatchers() error { for { select { case e := <-messageReceivedEvent: - messageID := string(e.MessageId[:]) - messageContent := e.Data - messageSender := string(e.Sender) - log.Info().Msgf("Message event received for message id: 0x%x", messageID) - log.Info().Msgf("Message event received with content: %+v", string(messageContent)) - log.Info().Msgf("Message event received with sender: 0x%x", messageSender[len(messageSender)-20:]) - lane.Dest.MessageReceivedWatcher.Store(messageID, messageContent) + processMessageReceivedEvent(lane, e) case <-lane.Context.Done(): return } @@ -3690,14 +3643,7 @@ func (lane *CCIPLane) StartEventWatchers() error { for { select { case e := <-reportBlessedEvent: - lane.Logger.Info().Msgf("TaggedRootBlessed event received for root %x", e.TaggedRoot.Root) - if e.TaggedRoot.CommitStore == lane.Dest.CommitStore.EthAddress { - lane.Dest.ReportBlessedWatcher.Store(e.TaggedRoot.Root, &contracts.LogInfo{ - BlockNumber: e.Raw.BlockNumber, - TxHash: e.Raw.TxHash, - }) - } - lane.Dest.ReportBlessedWatcher = testutils.DeleteNilEntriesFromMap(lane.Dest.ReportBlessedWatcher) + processTaggedRootBlessedEvent(lane, e) case <-lane.Context.Done(): return } @@ -3721,18 +3667,7 @@ func (lane *CCIPLane) StartEventWatchers() error { for { select { case e := <-execStateChangedEventLatest: - lane.Logger.Info().Msgf("Execution state changed event received for seq number %d", e.SequenceNumber) - lane.Dest.ExecStateChangedWatcher.Store(e.SequenceNumber, &contracts.EVM2EVMOffRampExecutionStateChanged{ - SequenceNumber: e.SequenceNumber, - MessageId: e.MessageId, - State: e.State, - ReturnData: e.ReturnData, - LogInfo: contracts.LogInfo{ - BlockNumber: e.Raw.BlockNumber, - TxHash: e.Raw.TxHash, - }, - }) - lane.Dest.ExecStateChangedWatcher = testutils.DeleteNilEntriesFromMap(lane.Dest.ExecStateChangedWatcher) + processExecutionStateChangedEvent(lane, e) case <-lane.Context.Done(): return } @@ -3741,6 +3676,282 @@ func (lane *CCIPLane) StartEventWatchers() error { return nil } +func processSendRequestedEvent(lane *CCIPLane, e *evm_2_evm_onramp.EVM2EVMOnRampCCIPSendRequested) { + lane.Logger.Info().Msgf("CCIPSendRequested event received for seq number %d", e.Message.SequenceNumber) + eventsForTx, ok := lane.Source.CCIPSendRequestedWatcher.Load(e.Raw.TxHash.Hex()) + if ok { + lane.Source.CCIPSendRequestedWatcher.Store(e.Raw.TxHash.Hex(), append(eventsForTx.([]*contracts.SendReqEventData), + &contracts.SendReqEventData{ + MessageId: e.Message.MessageId, + SequenceNumber: e.Message.SequenceNumber, + DataLength: len(e.Message.Data), + NoOfTokens: len(e.Message.TokenAmounts), + LogInfo: contracts.LogInfo{ + BlockNumber: e.Raw.BlockNumber, + TxHash: e.Raw.TxHash, + }, + Fee: e.Message.FeeTokenAmount, + })) + } else { + lane.Source.CCIPSendRequestedWatcher.Store(e.Raw.TxHash.Hex(), []*contracts.SendReqEventData{ + { + MessageId: e.Message.MessageId, + SequenceNumber: e.Message.SequenceNumber, + DataLength: len(e.Message.Data), + NoOfTokens: len(e.Message.TokenAmounts), + LogInfo: contracts.LogInfo{ + BlockNumber: e.Raw.BlockNumber, + TxHash: e.Raw.TxHash, + }, + Fee: e.Message.FeeTokenAmount, + }, + }) + } + + lane.Source.CCIPSendRequestedWatcher = testutils.DeleteNilEntriesFromMap(lane.Source.CCIPSendRequestedWatcher) +} + +func processReportAcceptedEvent(lane *CCIPLane, e *commit_store.CommitStoreReportAccepted) { + lane.Logger.Info().Interface("Interval", e.Report.Interval).Msgf("ReportAccepted event received") + for i := e.Report.Interval.Min; i <= e.Report.Interval.Max; i++ { + lane.Dest.ReportAcceptedWatcher.Store(i, &contracts.CommitStoreReportAccepted{ + Min: e.Report.Interval.Min, + Max: e.Report.Interval.Max, + MerkleRoot: e.Report.MerkleRoot, + LogInfo: contracts.LogInfo{ + BlockNumber: e.Raw.BlockNumber, + TxHash: e.Raw.TxHash, + }, + }) + } + + // Clean up nil entries + lane.Dest.ReportAcceptedWatcher = testutils.DeleteNilEntriesFromMap(lane.Dest.ReportAcceptedWatcher) +} + +func processTaggedRootBlessedEvent(lane *CCIPLane, e *rmn_contract.RMNContractTaggedRootBlessed) { + // Process the event + lane.Logger.Info().Msgf("TaggedRootBlessed event received for root %x", e.TaggedRoot.Root) + if e.TaggedRoot.CommitStore == lane.Dest.CommitStore.EthAddress { + lane.Dest.ReportBlessedWatcher.Store(e.TaggedRoot.Root, &contracts.LogInfo{ + BlockNumber: e.Raw.BlockNumber, + TxHash: e.Raw.TxHash, + }) + } + + // Clean up nil entries + lane.Dest.ReportBlessedWatcher = testutils.DeleteNilEntriesFromMap(lane.Dest.ReportBlessedWatcher) +} + +func processExecutionStateChangedEvent(lane *CCIPLane, e *evm_2_evm_offramp.EVM2EVMOffRampExecutionStateChanged) { + lane.Logger.Info().Msgf("Execution state changed event received for seq number %d", e.SequenceNumber) + lane.Dest.ExecStateChangedWatcher.Store(e.SequenceNumber, &contracts.EVM2EVMOffRampExecutionStateChanged{ + SequenceNumber: e.SequenceNumber, + MessageId: e.MessageId, + State: e.State, + ReturnData: e.ReturnData, + LogInfo: contracts.LogInfo{ + BlockNumber: e.Raw.BlockNumber, + TxHash: e.Raw.TxHash, + }, + }) + lane.Dest.ExecStateChangedWatcher = testutils.DeleteNilEntriesFromMap(lane.Dest.ExecStateChangedWatcher) +} + +func processMessageReceivedEvent(lane *CCIPLane, e *maybe_revert_message_receiver.MaybeRevertMessageReceiverMessageReceived) { + messageID := string(e.MessageId[:]) + messageContent := e.Data + messageSender := string(e.Sender) + log.Info().Msgf("Message event received for message id: 0x%x", messageID) + log.Info().Msgf("Message event received with content: %+v", string(messageContent)) + log.Info().Msgf("Message event received with sender: 0x%x", messageSender[len(messageSender)-20:]) + lane.Dest.MessageReceivedWatcher.Store(messageID, messageContent) +} + +func (lane *CCIPLane) StartEventWatchersPolling(sc *sentinel.SentinelCoordinator) error { + lane.Logger.Info().Msg("Starting event watchers, Polling") + if lane.Source.Common.ChainClient.GetNetworkConfig().FinalityDepth == 0 { + err := lane.Source.Common.ChainClient.PollFinality() + if err != nil { + return err + } + } + + sendReqEventSub, err := sc.Sentinel.Subscribe( + lane.SourceChain.GetChainID().Int64(), + lane.Source.OnRamp.EthAddress, + evm_2_evm_onramp.EVM2EVMOnRampCCIPSendRequested{}.Topic(), + ) + if err != nil { + log.Error().Err(err).Msg("error in setting up polling to CCIPSendRequested event") + return err + } + go func() { + for { + select { + case <-sc.Ctx.Done(): + return + case event, ok := <-sendReqEventSub: + if !ok { + lane.Logger.Info().Msg("Sentinel log channel closed. Exiting goroutine.") + return + } + typesLog, _ := sentinel.ConvertAPILogToTypesLog(event) + e, _ := lane.Source.OnRamp.Instance.Latest.EVM2EVMOnRampFilterer.ParseCCIPSendRequested(*typesLog) + processSendRequestedEvent(lane, e) + case <-lane.Context.Done(): + return + } + } + }() + + reportAcceptedSub, err := sc.Sentinel.Subscribe( + lane.DestChain.GetChainID().Int64(), + lane.Dest.CommitStore.EthAddress, + commit_store.CommitStoreReportAccepted{}.Topic(), + ) + if err != nil { + log.Error().Err(err).Msg("error in setting up polling to ReportAccepted event") + return errors.New("failed to subscribe to ReportAccepted event") + } + go func() { + for { + select { + case <-sc.Ctx.Done(): + return + case event, ok := <-reportAcceptedSub: + if !ok { + lane.Logger.Info().Msg("Sentinel log channel closed. Exiting goroutine.") + return + } + + // Convert the log and parse the ReportAccepted event + typesLog, err := sentinel.ConvertAPILogToTypesLog(event) + if err != nil { + lane.Logger.Error().Err(err).Msg("error converting Sentinel log to types.Log") + continue + } + e, err := lane.Dest.CommitStore.Instance.Latest.CommitStoreFilterer.ParseReportAccepted(*typesLog) + if err != nil { + lane.Logger.Error().Err(err).Msg("error parsing ReportAccepted event") + continue + } + processReportAcceptedEvent(lane, e) + case <-lane.Context.Done(): + return + } + } + }() + + if lane.Dest.Common.ARM != nil { + reportBlessedEventSub, err := sc.Sentinel.Subscribe( + lane.DestChain.GetChainID().Int64(), + lane.Dest.Common.ARM.EthAddress, + rmn_contract.RMNContractTaggedRootBlessed{}.Topic(), + ) + if err != nil { + log.Error().Err(err).Msg("error in setting up polling to TaggedRootBlessed event") + return errors.New("failed to subscribe to TaggedRootBlessed event") + } + + go func() { + for { + select { + case <-sc.Ctx.Done(): + return + case event, ok := <-reportBlessedEventSub: + if !ok { + lane.Logger.Info().Msg("Sentinel log channel closed. Exiting goroutine.") + return + } + + // Convert the log and parse the TaggedRootBlessed event + typesLog, err := sentinel.ConvertAPILogToTypesLog(event) + if err != nil { + lane.Logger.Error().Err(err).Msg("error converting Sentinel log to types.Log") + continue + } + e, err := lane.Dest.Common.ARM.Instance.RMNContractFilterer.ParseTaggedRootBlessed(*typesLog) + if err != nil { + lane.Logger.Error().Err(err).Msg("error parsing TaggedRootBlessed event") + continue + } + processTaggedRootBlessedEvent(lane, e) + case <-lane.Context.Done(): + return + } + } + }() + } + + execStateChangedEventSub, err := sc.Sentinel.Subscribe( + lane.DestChain.GetChainID().Int64(), + lane.Dest.OffRamp.EthAddress, + evm_2_evm_offramp.EVM2EVMOffRampExecutionStateChanged{}.Topic(), + ) + if err != nil { + log.Error().Err(err).Msg("error in setting up polling to ExecutionStateChanged event") + return errors.New("failed to subscribe to ExecutionStateChanged event") + } + + go func() { + for { + select { + case <-sc.Ctx.Done(): + return + case event, ok := <-execStateChangedEventSub: + if !ok { + lane.Logger.Info().Msg("Sentinel log channel closed. Exiting goroutine.") + return + } + + // Convert the log and parse the ExecutionStateChanged event + typesLog, err := sentinel.ConvertAPILogToTypesLog(event) + if err != nil { + lane.Logger.Error().Err(err).Msg("error converting Sentinel log to types.Log") + continue + } + e, err := lane.Dest.OffRamp.Instance.Latest.EVM2EVMOffRampFilterer.ParseExecutionStateChanged(*typesLog) + if err != nil { + lane.Logger.Error().Err(err).Msg("error parsing ExecutionStateChanged event") + continue + } + processExecutionStateChangedEvent(lane, e) + case <-lane.Context.Done(): + return + } + } + }() + messageReceivedSub, err := sc.Sentinel.Subscribe( + lane.DestChain.GetChainID().Int64(), + lane.Dest.ReceiverDapp.EthAddress, + maybe_revert_message_receiver.MaybeRevertMessageReceiverMessageReceived{}.Topic(), + ) + if err != nil { + log.Error().Err(err).Msg("error in setting up polling to MaybeRevertMessageReceiverMessageReceived event") + return err + } + go func() { + for { + select { + case <-sc.Ctx.Done(): + return + case event, ok := <-messageReceivedSub: + if !ok { + lane.Logger.Info().Msg("Sentinel log channel closed. Exiting goroutine.") + return + } + typesLog, _ := sentinel.ConvertAPILogToTypesLog(event) + e, _ := lane.Dest.ReceiverDapp.Instance.MaybeRevertMessageReceiverFilterer.ParseMessageReceived(*typesLog) + processMessageReceivedEvent(lane, e) + case <-lane.Context.Done(): + return + } + } + }() + return nil +} + func (lane *CCIPLane) CleanUp(clearFees bool) error { lane.Logger.Info().Msg("Cleaning up lane") if lane.Source.Common.ChainClient.GetNetworkConfig().FinalityDepth == 0 { diff --git a/integration-tests/ccip-tests/contracts/contract_deployer.go b/integration-tests/ccip-tests/contracts/contract_deployer.go index 13525422bed..13a20c670b8 100644 --- a/integration-tests/ccip-tests/contracts/contract_deployer.go +++ b/integration-tests/ccip-tests/contracts/contract_deployer.go @@ -809,7 +809,7 @@ func (e *CCIPContractsDeployer) DeployReceiverDapp(revert bool) ( return &ReceiverDapp{ client: e.evmClient, logger: e.logger, - instance: instance.(*maybe_revert_message_receiver.MaybeRevertMessageReceiver), + Instance: instance.(*maybe_revert_message_receiver.MaybeRevertMessageReceiver), EthAddress: *address, }, err } @@ -828,7 +828,7 @@ func (e *CCIPContractsDeployer) NewReceiverDapp(addr common.Address) ( return &ReceiverDapp{ client: e.evmClient, logger: e.logger, - instance: ins, + Instance: ins, EthAddress: addr, }, err } diff --git a/integration-tests/ccip-tests/contracts/contract_models.go b/integration-tests/ccip-tests/contracts/contract_models.go index 9ed310c3379..bcc209bd198 100644 --- a/integration-tests/ccip-tests/contracts/contract_models.go +++ b/integration-tests/ccip-tests/contracts/contract_models.go @@ -1078,7 +1078,7 @@ func (b *CommitStore) WatchReportAccepted(opts *bind.WatchOpts, acceptedEvent ch type ReceiverDapp struct { client blockchain.EVMClient logger *zerolog.Logger - instance *maybe_revert_message_receiver.MaybeRevertMessageReceiver + Instance *maybe_revert_message_receiver.MaybeRevertMessageReceiver EthAddress common.Address } @@ -1091,7 +1091,7 @@ func (rDapp *ReceiverDapp) ToggleRevert(revert bool) error { if err != nil { return fmt.Errorf("error getting transaction opts: %w", err) } - tx, err := rDapp.instance.SetRevert(opts, revert) + tx, err := rDapp.Instance.SetRevert(opts, revert) if err != nil { return fmt.Errorf("error setting revert: %w", err) } @@ -1106,8 +1106,8 @@ func (rDapp *ReceiverDapp) ToggleRevert(revert bool) error { // WatchMessageReceived watches for `MessageReceived` events from the ReceiverDapp contract. func (rDapp *ReceiverDapp) WatchMessageReceived(opts *bind.WatchOpts, messageReceivedEvent chan *maybe_revert_message_receiver.MaybeRevertMessageReceiverMessageReceived) (event.Subscription, error) { - if rDapp.instance != nil { - return rDapp.instance.WatchMessageReceived(opts, messageReceivedEvent) + if rDapp.Instance != nil { + return rDapp.Instance.WatchMessageReceived(opts, messageReceivedEvent) } newInstance, err := maybe_revert_message_receiver.NewMaybeRevertMessageReceiver(rDapp.EthAddress, wrappers.MustNewWrappedContractBackend(rDapp.client, nil)) @@ -1455,6 +1455,20 @@ func (r *Router) SetOnRamp(chainSelector uint64, onRamp common.Address) error { func (r *Router) CCIPSend(destChainSelector uint64, msg router.ClientEVM2AnyMessage, valueForNative *big.Int) (*types.Transaction, error) { opts, err := r.client.TransactionOpts(r.client.GetDefaultWallet()) + // print out opts + r.logger.Info(). + Str("from", opts.From.Hex()). + Str("nonce", fmt.Sprintf("%v", opts.Nonce)). + Str("value", fmt.Sprintf("%v", opts.Value)). + Str("gasPrice", fmt.Sprintf("%v", opts.GasPrice)). + Str("gasFeeCap", fmt.Sprintf("%v", opts.GasFeeCap)). + Str("gasTipCap", fmt.Sprintf("%v", opts.GasTipCap)). + Uint64("gasLimit", opts.GasLimit). + Str("accessList", fmt.Sprintf("%v", opts.AccessList)). + Str("context", fmt.Sprintf("%v", opts.Context)). + Bool("noSend", opts.NoSend). + Msg("TransactOpts") + if err != nil { return nil, fmt.Errorf("error getting transaction opts: %w", err) } diff --git a/integration-tests/ccip-tests/testconfig/README.md b/integration-tests/ccip-tests/testconfig/README.md index 9a9fa5129b1..b12d4e2d716 100644 --- a/integration-tests/ccip-tests/testconfig/README.md +++ b/integration-tests/ccip-tests/testconfig/README.md @@ -423,6 +423,32 @@ Example usage: TTL = "11h" ``` +### CCIP.Env.Logging + +Specifies the logging configuration for the test. Imported from [LoggingConfig](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/config/logging.go#L11) in chainlink-testing-framework. +Example usage: + +```toml +[CCIP.Env.Logging] +test_log_collect = false # if set to true will save logs even if test did not fail + +[CCIP.Env.Logging.LogStream] +# supported targets: file, loki, in-memory. if empty no logs will be persistet +log_targets = ["file"] +# context timeout for starting log producer and also time-frame for requesting logs +log_producer_timeout = "10s" +# number of retries before log producer gives up and stops listening to logs +log_producer_retry_limit = 10 + +[CCIP.Env.Logging.Loki] +tenant_id = "..." +endpoint = "https://loki...." + +[CCIP.Env.Logging.Grafana] +base_url = "https://grafana..../" +dashboard_url = "/d/6vjVx-1V8/ccip-long-running-tests" +``` + ### CCIP.Env.Lane.LeaderLaneEnabled Specifies whether to enable the leader lane feature. This setting is only applicable for new deployments. diff --git a/integration-tests/ccip-tests/testconfig/global.go b/integration-tests/ccip-tests/testconfig/global.go index 8866d31705a..eb102633ebf 100644 --- a/integration-tests/ccip-tests/testconfig/global.go +++ b/integration-tests/ccip-tests/testconfig/global.go @@ -175,6 +175,90 @@ type Common struct { func (p *Common) ReadFromEnvVar() error { logger := logging.GetTestLogger(nil) + lokiTenantID := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_TENANT_ID_ENV) + if lokiTenantID != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.TenantId", ctfconfig.E2E_TEST_LOKI_TENANT_ID_ENV) + p.Logging.Loki.TenantId = &lokiTenantID + } + + lokiEndpoint := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_ENDPOINT_ENV) + if lokiEndpoint != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.Endpoint", ctfconfig.E2E_TEST_LOKI_ENDPOINT_ENV) + p.Logging.Loki.Endpoint = &lokiEndpoint + } + + lokiBasicAuth := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_BASIC_AUTH_ENV) + if lokiBasicAuth != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.BasicAuth", ctfconfig.E2E_TEST_LOKI_BASIC_AUTH_ENV) + p.Logging.Loki.BasicAuth = &lokiBasicAuth + } + + lokiBearerToken := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_BEARER_TOKEN_ENV) + if lokiBearerToken != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Loki == nil { + p.Logging.Loki = &ctfconfig.LokiConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Loki.BearerToken", ctfconfig.E2E_TEST_LOKI_BEARER_TOKEN_ENV) + p.Logging.Loki.BearerToken = &lokiBearerToken + } + + grafanaBaseURL := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_GRAFANA_BASE_URL_ENV) + if grafanaBaseURL != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Grafana == nil { + p.Logging.Grafana = &ctfconfig.GrafanaConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Grafana.BaseUrl", ctfconfig.E2E_TEST_GRAFANA_BASE_URL_ENV) + p.Logging.Grafana.BaseUrl = &grafanaBaseURL + } + + grafanaDashboardURL := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_GRAFANA_DASHBOARD_URL_ENV) + if grafanaDashboardURL != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Grafana == nil { + p.Logging.Grafana = &ctfconfig.GrafanaConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Grafana.DashboardUrl", ctfconfig.E2E_TEST_GRAFANA_DASHBOARD_URL_ENV) + p.Logging.Grafana.DashboardUrl = &grafanaDashboardURL + } + + grafanaBearerToken := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_GRAFANA_BEARER_TOKEN_ENV) + if grafanaBearerToken != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.Grafana == nil { + p.Logging.Grafana = &ctfconfig.GrafanaConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.Grafana.BearerToken", ctfconfig.E2E_TEST_GRAFANA_BEARER_TOKEN_ENV) + p.Logging.Grafana.BearerToken = &grafanaBearerToken + } + selectedNetworks := ctfconfig.MustReadEnvVar_Strings(ctfconfig.E2E_TEST_SELECTED_NETWORK_ENV, ",") if len(selectedNetworks) > 0 { if p.Network == nil { diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index be736dca1db..d78961dadc3 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -37,6 +37,8 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/lib/k8s/environment" "github.com/smartcontractkit/chainlink-testing-framework/lib/networks" "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/testcontext" + "github.com/smartcontractkit/chainlink-testing-framework/sentinel" + "github.com/smartcontractkit/chainlink-testing-framework/sentinel/blockchain_client_wrapper" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" integrationactions "github.com/smartcontractkit/chainlink/integration-tests/actions" @@ -47,6 +49,7 @@ import ( ccipconfig "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testconfig" "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testreporters" testutils "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/utils" + "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" ) @@ -577,6 +580,7 @@ type CCIPTestSetUpOutputs struct { Balance *actions.BalanceSheet BootstrapAdded *atomic.Bool JobAddGrp *errgroup.Group + SC *sentinel.SentinelCoordinator } func (o *CCIPTestSetUpOutputs) AddToLanes(lane *BiDirectionalLaneConfig) { @@ -888,6 +892,17 @@ func (o *CCIPTestSetUpOutputs) StartEventWatchers() { } } +func (o *CCIPTestSetUpOutputs) StartEventWatchersPolling() { + for _, lane := range o.ReadLanes() { + err := lane.ForwardLane.StartEventWatchersPolling(o.SC) + require.NoError(o.Cfg.Test, err) + if lane.ReverseLane != nil { + err = lane.ReverseLane.StartEventWatchersPolling(o.SC) + require.NoError(o.Cfg.Test, err) + } + } +} + func (o *CCIPTestSetUpOutputs) WaitForPriceUpdates() { t := o.Cfg.Test priceUpdateGrp, _ := errgroup.WithContext(o.SetUpContext) @@ -1215,7 +1230,21 @@ func CCIPDefaultTestSetUp( } // start event watchers for all lanes - setUpArgs.StartEventWatchers() + if useWebSocket(chainClientByChainID) { + setUpArgs.StartEventWatchers() + } else { + setUpArgs.SC = sentinel.NewSentinelCoordinator(*lggr) + err := setUpArgs.addChains() + require.NoError(t, err, "error adding chain to Sentinel") + setUpArgs.StartEventWatchersPolling() + t.Cleanup(func() { + if setUpArgs.SC != nil { + lggr.Info().Msg("Closing Sentinel") + setUpArgs.SC.Sentinel.Close() + } + }) + } + // now that lane configs are already dumped to file, we can clean up the lane config map setUpArgs.LaneConfig = nil setUpArgs.TearDown = func() error { @@ -1240,6 +1269,47 @@ func CCIPDefaultTestSetUp( return setUpArgs } +func useWebSocket(chainClientByChainID map[int64]blockchain.EVMClient) bool { + for _, c := range chainClientByChainID { + if !c.GetEthClient().Client().SupportsSubscriptions() { + return false + } + } + return true +} + +func (o *CCIPTestSetUpOutputs) addChains() error { + for _, lane := range o.ReadLanes() { + // Add both forward and reverse lanes + err := o.addChainToSentinel(lane.ForwardLane.SourceChain) + if err != nil { + return err + } + err = o.addChainToSentinel(lane.ForwardLane.DestChain) + if err != nil { + return err + } + } + return nil +} + +// addChainToSentinel is a helper function to add a chain to Sentinel +func (o *CCIPTestSetUpOutputs) addChainToSentinel(chain blockchain.EVMClient) error { + blockchainClient := blockchain_client_wrapper.NewGethClientWrapper(chain.GetEthClient()) + + // Define the chain poller service configuration + addChainConfig := sentinel.AddChainConfig{ + ChainID: chain.GetChainID().Int64(), + PollInterval: 30 * time.Second, + BlockchainClient: blockchainClient, + } + + // Add the chain to Sentinel + err := o.SC.Sentinel.AddChain(addChainConfig) + + return err +} + // CreateEnvironment creates the environment for the test and registers the test clean-up function to tear down the set-up environment // It returns the map of chainID to EVMClient func (o *CCIPTestSetUpOutputs) CreateEnvironment( diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 5296bf8c9af..3dabf60b532 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -54,8 +54,9 @@ require ( github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 - github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.22 + github.com/smartcontractkit/chainlink-testing-framework/lib v1.51.0 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 + github.com/smartcontractkit/chainlink-testing-framework/sentinel v0.1.2 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 6a8c456dca1..4a5549effb3 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1465,10 +1465,12 @@ github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7 h1:E7k5 github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7/go.mod h1:WYxCxAWpeXEHfhB0GaiV2sj21Ooh9r/Nf7tzmJgAibs= github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 h1:GDGrC5OGiV0RyM1znYWehSQXyZQWTOzrEeJRYmysPCE= github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2/go.mod h1:DsT43c1oTBmp3iQkMcoZOoKThwZvt8X3Pz6UmznJ4GY= -github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.22 h1:W3doYLVoZN8VwJb/kAZsbDjW+6cgZPgNTcQHJUH9JrA= -github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.22/go.mod h1:70JLBXQncNHyW63ik4PvPQGjQGZ1xK67MKrDanVAk2w= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.51.0 h1:z7t2OhfE32KK4r5Nt3U0hOnbRwOwIbJs8i7kqKvjAA0= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.51.0/go.mod h1:y6pVvAT/R+YGocAqoQIat+AEaZz2Jdmj/0uUBmwvLCU= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= +github.com/smartcontractkit/chainlink-testing-framework/sentinel v0.1.2 h1:ihRlWrii5nr4RUuMu1hStTbwFvVuHUDoQQwXmCU5IdQ= +github.com/smartcontractkit/chainlink-testing-framework/sentinel v0.1.2/go.mod h1:J1Za5EuI/vWDsQSIh6qbPXlVvuEhmHmnvLQBN0XVxqA= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10 h1:Yf+n3T/fnUWcYyfe7bsygV4sWAkNo0QhN58APJFIKIc= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10/go.mod h1:05duR85P8YHuIfIkA7sn2bvrhKo/pDpFKV2rliYHNOo= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 h1:7bCdbTUWzyczQg+kwHCxlx6y07zE8HNB8+ntTne6qd8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index f4e8efc4df3..c4b1daec25e 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -31,7 +31,7 @@ require ( github.com/smartcontractkit/chainlink-ccip v0.0.0-20250205140756-e0f1a86dfdb3 github.com/smartcontractkit/chainlink-common v0.4.2-0.20250130202959-6f1f48342e36 github.com/smartcontractkit/chainlink-integrations/evm v0.0.0-20250206144234-88579df97ecd - github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.22 + github.com/smartcontractkit/chainlink-testing-framework/lib v1.51.0 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20241009055228-33d0c0bf38de diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 97425850903..93346d9c39b 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1452,8 +1452,8 @@ github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7 h1:E7k5 github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.7/go.mod h1:WYxCxAWpeXEHfhB0GaiV2sj21Ooh9r/Nf7tzmJgAibs= github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2 h1:GDGrC5OGiV0RyM1znYWehSQXyZQWTOzrEeJRYmysPCE= github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.2/go.mod h1:DsT43c1oTBmp3iQkMcoZOoKThwZvt8X3Pz6UmznJ4GY= -github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.22 h1:W3doYLVoZN8VwJb/kAZsbDjW+6cgZPgNTcQHJUH9JrA= -github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.22/go.mod h1:70JLBXQncNHyW63ik4PvPQGjQGZ1xK67MKrDanVAk2w= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.51.0 h1:z7t2OhfE32KK4r5Nt3U0hOnbRwOwIbJs8i7kqKvjAA0= +github.com/smartcontractkit/chainlink-testing-framework/lib v1.51.0/go.mod h1:y6pVvAT/R+YGocAqoQIat+AEaZz2Jdmj/0uUBmwvLCU= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10 h1:Yf+n3T/fnUWcYyfe7bsygV4sWAkNo0QhN58APJFIKIc=