diff --git a/config/default-config.yml b/config/default-config.yml index 620b3c8361c..2602bb5689d 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -140,38 +140,66 @@ network-config: # The size of the queue for notifications about invalid RPC messages notification-cache-size: 10_000 validation: # RPC control message validation inspector configs - # Rpc validation inspector number of pool workers - workers: 5 - # The size of the queue used by worker pool for the control message validation inspector - queue-size: 100 - # The max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated - message-max-sample-size: 1000 - # Max number of control messages in a sample to be inspected when inspecting GRAFT and PRUNE message types. If the total number of control messages (GRAFT or PRUNE) - # exceeds this max sample size then the respective message will be truncated before being processed. - graft-and-prune-message-max-sample-size: 1000 - # The threshold at which an error will be returned if the number of invalid RPC messages exceeds this value - error-threshold: 500 - ihave: # Max number of ihave messages in a sample to be inspected. If the number of ihave messages exceeds this configured value - # the control message ihaves will be truncated to the max sample size. This sample is randomly selected. + inspection-queue: + # Rpc validation inspector number of pool workers + workers: 5 + # The size of the queue used by worker pool for the control message validation inspector + queue-size: 100 + publish-messages: + # The maximum number of messages in a single RPC message that are randomly sampled for async inspection. + # When the size of a single RPC message exceeds this threshold, a random sample is taken for inspection, but the RPC message is not truncated. max-sample-size: 1000 - # Max number of ihave message ids in a sample to be inspected per ihave. Each ihave message includes a list of message ids - # each. If the size of the message ids list for a single ihave message exceeds the configured max message id sample size the list of message ids will be truncated. - max-message-id-sample-size: 1000 + # The threshold at which an error will be returned if the number of invalid RPC messages exceeds this value + error-threshold: 500 + graft-and-prune: + # The maximum number of GRAFT or PRUNE messages in a single RPC message. + # When the total number of GRAFT or PRUNE messages in a single RPC message exceeds this threshold, + # a random sample of GRAFT or PRUNE messages will be taken and the RPC message will be truncated to this sample size. + message-count-threshold: 1000 + # Maximum number of total duplicate topic ids in a single GRAFT or PRUNE message, ideally this should be 0 but we allow for some tolerance + # to avoid penalizing peers that are not malicious but are misbehaving due to bugs or other issues. + # A topic id is considered duplicate if it appears more than once in a single GRAFT or PRUNE message. + duplicate-topic-id-threshold: 50 + ihave: + # The maximum allowed number of iHave messages in a single RPC message. + # Each iHave message represents the list of message ids. When the total number of iHave messages + # in a single RPC message exceeds this threshold, a random sample of iHave messages will be taken and the RPC message will be truncated to this sample size. + # The sample size is equal to the configured message-count-threshold. + message-count-threshold: 1000 + # The maximum allowed number of message ids in a single iHave message. + # Each iHave message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + # that can be included in a single iHave message. When the total number of message ids in a single iHave message exceeds this threshold, + # a random sample of message ids will be taken and the iHave message will be truncated to this sample size. + # The sample size is equal to the configured message-id-count-threshold. + message-id-count-threshold: 1000 + # The tolerance threshold for having duplicate topics in an iHave message under inspection. + # When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. + # Note that a topic ID is counted as a duplicate only if it is repeated more than once. + duplicate-topic-id-threshold: 50 + # Threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. + # When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + # Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once + # within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + duplicate-message-id-threshold: 100 iwant: - # Max number of iwant messages in a sample to be inspected. If the total number of iWant control messages - # exceeds this max sample size then the respective message will be truncated before being processed. - max-sample-size: 1000 - # Max number of iwant message ids in a sample to be inspected per iwant. Each iwant message includes a list of message ids - # each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - max-message-id-sample-size: 1000 - # The allowed threshold of iWant messages received without a corresponding tracked iHave message that was sent. If the cache miss threshold is exceeded an - # invalid control message notification is disseminated and the sender will be penalized. - cache-miss-threshold: .5 - # The iWants size at which message id cache misses will be checked. - cache-miss-check-size: 1000 - # The max allowed duplicate message IDs in a single iWant control message. If the duplicate message threshold is exceeded an invalid control message - # notification is disseminated and the sender will be penalized. - duplicate-message-id-threshold: .15 + # The maximum allowed number of iWant messages in a single RPC message. + # Each iWant message represents the list of message ids. When the total number of iWant messages + # in a single RPC message exceeds this threshold, a random sample of iWant messages will be taken and the RPC message will be truncated to this sample size. + # The sample size is equal to the configured message-count-threshold. + message-count-threshold: 1000 + # The maximum allowed number of message ids in a single iWant message. + # Each iWant message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + # that can be included in a single iWant message. When the total number of message ids in a single iWant message exceeds this threshold, + # a random sample of message ids will be taken and the iWant message will be truncated to this sample size. + # The sample size is equal to the configured message-id-count-threshold. + message-id-count-threshold: 1000 + # The allowed threshold of iWant messages received without a corresponding tracked iHave message that was sent. + # If the cache miss threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. + cache-miss-threshold: 500 + # The max allowed number of duplicate message ids in a single iwant message. + # Note that ideally there should be no duplicate message ids in a single iwant message but + # we allow for some tolerance to avoid penalizing peers that are not malicious + duplicate-message-id-threshold: 100 cluster-prefixed-messages: # Cluster prefixed control message validation configs # The size of the cache used to track the amount of cluster prefixed topics received by peers diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 6a3e168f0f4..0f68de8a4d7 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -3,6 +3,7 @@ package rpc_inspector import ( "context" "fmt" + "math" "os" "testing" "time" @@ -46,7 +47,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation messageCount := 100 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 controlMessageCount := int64(1) count := atomic.NewUint64(0) @@ -179,9 +180,11 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 - messageCount := 10 + // sets the message count to the max of the duplicate topic id threshold for graft and prune control messages to ensure + // a successful attack + messageCount := int(math.Max(float64(inspectorConfig.IHave.DuplicateTopicIdThreshold), float64(inspectorConfig.GraftPrune.DuplicateTopicIdThreshold))) + 2 controlMessageCount := int64(1) count := atomic.NewInt64(0) @@ -289,7 +292,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 count := atomic.NewInt64(0) done := make(chan struct{}) @@ -301,7 +304,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { notification, ok := args[0].(*p2p.InvCtrlMsgNotif) require.True(t, ok) require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "IsClusterPrefixed is expected to be false, no RPC with cluster prefixed topic sent in this test") - require.True(t, validation.IsDuplicateTopicErr(notification.Error)) + require.True(t, validation.IsDuplicateMessageIDErr(notification.Error)) require.Equal(t, spammer.SpammerNode.ID(), notification.PeerID) require.True(t, notification.MsgType == p2pmsg.CtrlMsgIHave, @@ -361,21 +364,25 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { validationInspector.Start(signalerCtx) nodes := []p2p.LibP2PNode{victimNode, spammer.SpammerNode} startNodesAndEnsureConnected(t, signalerCtx, nodes, sporkID) + // to suppress peers provider not set + p2ptest.RegisterPeerProviders(t, nodes) spammer.Start(t) defer stopComponents(t, cancel, nodes, validationInspector) // generate 2 control messages with iHaves for different topics - ihaveCtlMsgs1 := spammer.GenerateCtlMessages(1, p2ptest.WithIHave(1, 1, pushBlocks.String())) - ihaveCtlMsgs2 := spammer.GenerateCtlMessages(1, p2ptest.WithIHave(1, 1, reqChunks.String())) - - // duplicate message ids for a single topic is invalid and will cause an error - ihaveCtlMsgs1[0].Ihave[0].MessageIDs = append(ihaveCtlMsgs1[0].Ihave[0].MessageIDs, ihaveCtlMsgs1[0].Ihave[0].MessageIDs[0]) - // duplicate message ids across different topics is valid - ihaveCtlMsgs2[0].Ihave[0].MessageIDs[0] = ihaveCtlMsgs1[0].Ihave[0].MessageIDs[0] + messageIdCount := inspectorConfig.IHave.DuplicateMessageIdThreshold + 2 + messageIds := unittest.IdentifierListFixture(1) + for i := 0; i < messageIdCount; i++ { + messageIds = append(messageIds, messageIds[0]) + } + // prepares 2 control messages with iHave messages for different topics with duplicate message IDs + ihaveCtlMsgs1 := spammer.GenerateCtlMessages( + 1, + p2ptest.WithIHaveMessageIDs(messageIds.Strings(), pushBlocks.String()), + p2ptest.WithIHaveMessageIDs(messageIds.Strings(), reqChunks.String())) // start spamming the victim peer spammer.SpamControlMessage(t, victimNode, ihaveCtlMsgs1) - spammer.SpamControlMessage(t, victimNode, ihaveCtlMsgs2) unittest.RequireCloseBefore(t, done, 5*time.Second, "failed to inspect RPC messages on time") // ensure we receive the expected number of invalid control message notifications @@ -393,7 +400,7 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { // set hard threshold to 0 so that in the case of invalid cluster ID // we force the inspector to return an error inspectorConfig.ClusterPrefixedMessage.HardThreshold = 0 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. @@ -502,7 +509,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 controlMessageCount := int64(10) count := atomic.NewInt64(0) @@ -590,7 +597,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 controlMessageCount := int64(10) count := atomic.NewInt64(0) @@ -676,7 +683,7 @@ func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { // set hard threshold to 0 so that in the case of invalid cluster ID // we force the inspector to return an error inspectorConfig.ClusterPrefixedMessage.HardThreshold = 0 - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 // SafetyThreshold < messageCount < HardThreshold ensures that the RPC message will be further inspected and topic IDs will be checked // restricting the message count to 1 allows us to only aggregate a single error when the error is logged in the inspector. @@ -771,11 +778,9 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { flowConfig, err := config.DefaultConfig() require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - // force all cache miss checks - inspectorConfig.IWant.CacheMissCheckSize = 1 - inspectorConfig.NumberOfWorkers = 1 - inspectorConfig.IWant.CacheMissThreshold = .5 // set cache miss threshold to 50% - messageCount := 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 + inspectorConfig.IWant.CacheMissThreshold = 10 + messageCount := 10 controlMessageCount := int64(1) cacheMissThresholdNotifCount := atomic.NewUint64(0) done := make(chan struct{}) @@ -875,7 +880,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { flowConfig, err := config.DefaultConfig() require.NoError(t, err) inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation - inspectorConfig.NumberOfWorkers = 1 + inspectorConfig.InspectionQueue.NumberOfWorkers = 1 idProvider := mock.NewIdentityProvider(t) spammer := corruptlibp2p.NewGossipSubRouterSpammer(t, sporkID, role, idProvider) @@ -968,7 +973,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { topicProvider.UpdateTopics(topics) // after 7 errors encountered disseminate a notification - inspectorConfig.MessageErrorThreshold = 6 + inspectorConfig.PublishMessages.ErrorThreshold = 6 require.NoError(t, err) corruptInspectorFunc := corruptlibp2p.CorruptInspectorFunc(validationInspector) @@ -1087,12 +1092,14 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { graftCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, unknownTopic.String())) graftCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, malformedTopic.String())) graftCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, invalidSporkIDTopic.String())) - graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(3, duplicateTopic.String())) + graftCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), // sets duplicate to +2 above the threshold to ensure that the victim node will penalize the spammer node + p2ptest.WithGraft(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2, duplicateTopic.String())) pruneCtlMsgsWithUnknownTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithPrune(spamRpcCount, unknownTopic.String())) pruneCtlMsgsWithMalformedTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithPrune(spamRpcCount, malformedTopic.String())) pruneCtlMsgsInvalidSporkIDTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithGraft(spamRpcCount, invalidSporkIDTopic.String())) - pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), p2ptest.WithPrune(3, duplicateTopic.String())) + pruneCtlMsgsDuplicateTopic := spammer.GenerateCtlMessages(int(spamCtrlMsgCount), // sets duplicate to +2 above the threshold to ensure that the victim node will penalize the spammer node + p2ptest.WithPrune(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2, duplicateTopic.String())) // start spamming the victim peer spammer.SpamControlMessage(t, victimNode, graftCtlMsgsWithUnknownTopic) diff --git a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go index 8d6cfec2cef..c43b7435f55 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go @@ -206,15 +206,20 @@ func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { conf, err := config.DefaultConfig() require.NoError(t, err) // overcompensate for RPC truncation - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize = 10000 - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize = 10000 - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize = 10000 - conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageCountThreshold = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageIdCountThreshold = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageCountThreshold = 10000 + conf.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.MessageIdCountThreshold = 10000 // we override the decay interval to 1 second so that the score is updated within 1 second intervals. conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.DecayInterval = 1 * time.Second // score tracer interval is set to 500 milliseconds to speed up the test, it should be shorter than the heartbeat interval (1 second) of gossipsub to catch the score updates in time. conf.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 500 * time.Millisecond + // relaxing the scoring parameters to fit the test scenario. + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyDecay = 0.99 + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyThreshold = 10 + conf.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.Behaviour.PenaltyWeight = -1 + ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. diff --git a/module/metrics.go b/module/metrics.go index 9713e022430..4315d6c017b 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -276,6 +276,83 @@ type GossipSubRpcValidationInspectorMetrics interface { // messageType: the type of the control message that was truncated // diff: the number of control messages truncated. OnControlMessagesTruncated(messageType p2pmsg.ControlMessageType, diff int) + + // OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on iWant messages at the end of the async inspection iWants + // across one RPC, regardless of the result of the inspection. + // + // duplicateCount: the total number of duplicate message ids received by the node on the iWant messages at the end of the async inspection of the RPC. + // cacheMissCount: the total number of cache miss message ids received by the node on the iWant message at the end of the async inspection of the RPC. + OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) + + // OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total number of duplicate message ids + // received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. + OnIWantDuplicateMessageIdsExceedThreshold() + + // OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total + // number of cache miss message ids received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. + OnIWantCacheMissMessageIdsExceedThreshold() + + // OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages of a single RPC, regardless of the result of the inspection. + // It tracks the number of duplicate topic ids and duplicate message ids received by the node on the iHave messages of that single RPC at the end of the async inspection iHaves. + // Args: + // + // duplicateTopicIds: the total number of duplicate topic ids received by the node on the iHave messages at the end of the async inspection of the RPC. + // duplicateMessageIds: the number of duplicate message ids received by the node on the iHave messages at the end of the async inspection of the RPC. + OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) + + // OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate topic ids + // received by the node on the iHave messages of that RPC exceeding the threshold, which results in a misbehaviour report. + OnIHaveDuplicateTopicIdsExceedThreshold() + + // OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate message ids + // received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. + OnIHaveDuplicateMessageIdsExceedThreshold() + + // OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message type on a single RPC failed due to an invalid topic id. + // Args: + // - messageType: the type of the control message that was truncated. + OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) + + // OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message type on a single RPC failed due to active cluster ids not set inspection failure. + // This is not causing a misbehaviour report. + OnActiveClusterIDsNotSetErr() + + // OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message type on a single RPC failed due to unstaked peer inspection failure. + // This is not causing a misbehaviour report. + OnUnstakedPeerInspectionFailed() + + // OnInvalidControlMessageNotificationSent tracks the number of times that the async inspection of a control message failed and resulted in dissemination of an invalid control message was sent. + OnInvalidControlMessageNotificationSent() + + // OnPublishMessagesInspectionErrorExceedsThreshold tracks the number of times that async inspection of publish messages failed due to the number of errors. + OnPublishMessagesInspectionErrorExceedsThreshold() + + // OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of prune messages for an RPC failed due to the number of duplicate topic ids + // received by the node on prune messages of the same RPC excesses threshold, which results in a misbehaviour report. + OnPruneDuplicateTopicIdsExceedThreshold() + + // OnPruneMessageInspected is called at the end of the async inspection of prune messages of the RPC, regardless of the result of the inspection. + // Args: + // duplicateTopicIds: the number of duplicate topic ids received by the node on the prune messages of the RPC at the end of the async inspection prunes. + OnPruneMessageInspected(duplicateTopicIds int) + + // OnGraftDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of the graft messages of a single RPC failed due to the number of duplicate topic ids + // received by the node on graft messages of the same RPC excesses threshold, which results in a misbehaviour report. + OnGraftDuplicateTopicIdsExceedThreshold() + + // OnGraftMessageInspected is called at the end of the async inspection of graft messages of a single RPC, regardless of the result of the inspection. + // Args: + // duplicateTopicIds: the number of duplicate topic ids received by the node on the graft messages at the end of the async inspection of a single RPC. + OnGraftMessageInspected(duplicateTopicIds int) + + // OnPublishMessageInspected is called at the end of the async inspection of publish messages of a single RPC, regardless of the result of the inspection. + // It tracks the total number of errors detected during the async inspection of the rpc together with their individual breakdown. + // Args: + // - errCount: the number of errors that occurred during the async inspection of publish messages. + // - invalidTopicIdsCount: the number of times that an invalid topic id was detected during the async inspection of publish messages. + // - invalidSubscriptionsCount: the number of times that an invalid subscription was detected during the async inspection of publish messages. + // - invalidSendersCount: the number of times that an invalid sender was detected during the async inspection of publish messages. + OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) } // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. diff --git a/module/metrics/gossipsub_rpc_validation_inspector.go b/module/metrics/gossipsub_rpc_validation_inspector.go index 69706fa0083..6b79e8c477d 100644 --- a/module/metrics/gossipsub_rpc_validation_inspector.go +++ b/module/metrics/gossipsub_rpc_validation_inspector.go @@ -21,6 +21,7 @@ type GossipSubRpcValidationInspectorMetrics struct { rpcCtrlMsgInAsyncPreProcessingGauge prometheus.Gauge rpcCtrlMsgAsyncProcessingTimeHistogram prometheus.Histogram rpcCtrlMsgTruncation prometheus.HistogramVec + ctrlMsgInvalidTopicIdCount prometheus.CounterVec receivedIWantMsgCount prometheus.Counter receivedIWantMsgIDsHistogram prometheus.Histogram receivedIHaveMsgCount prometheus.Counter @@ -29,6 +30,38 @@ type GossipSubRpcValidationInspectorMetrics struct { receivedGraftCount prometheus.Counter receivedPublishMessageCount prometheus.Counter incomingRpcCount prometheus.Counter + + // graft inspection + graftDuplicateTopicIdsHistogram prometheus.Histogram + graftDuplicateTopicIdsExceedThresholdCount prometheus.Counter + + // prune inspection + pruneDuplicateTopicIdsHistogram prometheus.Histogram + pruneDuplicateTopicIdsExceedThresholdCount prometheus.Counter + + // iHave inspection + iHaveDuplicateMessageIdHistogram prometheus.Histogram + iHaveDuplicateTopicIdHistogram prometheus.Histogram + iHaveDuplicateMessageIdExceedThresholdCount prometheus.Counter + iHaveDuplicateTopicIdExceedThresholdCount prometheus.Counter + + // iWant inspection + iWantDuplicateMessageIdHistogram prometheus.Histogram + iWantCacheMissHistogram prometheus.Histogram + iWantDuplicateMessageIdExceedThresholdCount prometheus.Counter + iWantCacheMissMessageIdExceedThresholdCount prometheus.Counter + + // inspection result + errActiveClusterIdsNotSetCount prometheus.Counter + errUnstakedPeerInspectionFailedCount prometheus.Counter + invalidControlMessageNotificationSentCount prometheus.Counter + + // publish messages + publishMessageInspectionErrExceedThresholdCount prometheus.Counter + publishMessageInvalidSenderCountHistogram prometheus.Histogram + publishMessageInvalidSubscriptionsHistogram prometheus.Histogram + publishMessageInvalidTopicIdHistogram prometheus.Histogram + publishMessageInspectedErrHistogram prometheus.Histogram } var _ module.GossipSubRpcValidationInspectorMetrics = (*GossipSubRpcValidationInspectorMetrics)(nil) @@ -80,7 +113,7 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_iwant_total", - Help: "number of received iwant messages from gossipsub protocol", + Help: "total number of received iwant messages from gossipsub protocol", }) gc.receivedIWantMsgIDsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ @@ -94,28 +127,185 @@ func NewGossipSubRPCValidationInspectorMetrics(prefix string) *GossipSubRpcValid Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_graft_total", - Help: "number of received graft messages from gossipsub protocol", + Help: "total number of received graft messages from gossipsub protocol", }) gc.receivedPruneCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_prune_total", - Help: "number of received prune messages from gossipsub protocol", + Help: "total number of received prune messages from gossipsub protocol", }) gc.receivedPublishMessageCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_received_publish_message_total", - Help: "number of received publish messages from gossipsub protocol", + Help: "total number of received publish messages from gossipsub protocol", }) gc.incomingRpcCount = promauto.NewCounter(prometheus.CounterOpts{ Namespace: namespaceNetwork, Subsystem: subsystemGossip, Name: gc.prefix + "gossipsub_incoming_rpc_total", - Help: "number of incoming rpc messages from gossipsub protocol", + Help: "total number of incoming rpc messages from gossipsub protocol", + }) + + gc.iHaveDuplicateMessageIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Buckets: []float64{1, 100, 1000}, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids_count", + Help: "number of duplicate message ids received from gossipsub protocol during the async inspection of a single RPC", + }) + + gc.iHaveDuplicateTopicIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Buckets: []float64{1, 100, 1000}, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids_count", + Help: "number of duplicate topic ids received from gossipsub protocol during the async inspection of a single RPC", + }) + + gc.iHaveDuplicateMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_message_ids_exceed_threshold_total", + Help: "total number of times that the async inspection of iHave messages failed due to the number of duplicate message ids exceeding the threshold", + }) + + gc.iHaveDuplicateTopicIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_ihave_duplicate_topic_ids_exceed_threshold_total", + Help: "total number of times that the async inspection of iHave messages failed due to the number of duplicate topic ids exceeding the threshold", + }) + + gc.iWantDuplicateMessageIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of duplicate message ids received from gossipsub protocol during the async inspection of a single RPC", + }) + + gc.iWantCacheMissHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids_count", + Buckets: []float64{1, 100, 1000}, + Help: "total number of cache miss message ids received from gossipsub protocol during the async inspection of a single RPC", + }) + + gc.iWantDuplicateMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_duplicate_message_ids_exceed_threshold_total", + Help: "total number of times that the async inspection of iWant messages failed due to the number of duplicate message ids ", + }) + + gc.iWantCacheMissMessageIdExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_iwant_cache_miss_message_ids_exceed_threshold_total", + Help: "total number of times that the async inspection of iWant messages failed due to the number of cache miss message ids ", + }) + + gc.ctrlMsgInvalidTopicIdCount = *promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "control_message_invalid_topic_id_total", + Help: "total number of control messages with invalid topic id received from gossipsub protocol during the async inspection", + }, []string{LabelMessage}) + + gc.errActiveClusterIdsNotSetCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "active_cluster_ids_not_inspection_error_total", + Help: "total number of inspection errors due to active cluster ids not set inspection failure", + }) + + gc.errUnstakedPeerInspectionFailedCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "unstaked_peer_inspection_error_total", + Help: "total number of inspection errors due to unstaked peer inspection failure", + }) + + gc.invalidControlMessageNotificationSentCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "invalid_control_message_notification_sent_total", + Help: "number of invalid control message notifications (i.e., misbehavior report) sent due to async inspection of rpcs failure", + }) + + gc.graftDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of duplicate topic ids on graft messages of a single RPC during the async inspection, regardless of the result of the inspection", + }) + + gc.graftDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_graft_duplicate_topic_ids_exceed_threshold_total", + Help: "number of times that the async inspection of graft messages of an rpc failed due to the number of duplicate topic ids exceeding the threshold", + }) + + gc.pruneDuplicateTopicIdsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Buckets: []float64{1, 100, 1000}, + Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids_count", + Help: "number of duplicate topic ids on prune messages of a single RPC during the async inspection, regardless of the result of the inspection", + }) + + gc.pruneDuplicateTopicIdsExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_prune_duplicate_topic_ids_exceed_threshold_total", + Help: "number of times that the async inspection of prune messages failed due to the number of duplicate topic ids exceeding the threshold", + }) + + gc.publishMessageInspectedErrHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "publish_message_inspected_error_count", + Buckets: []float64{10, 100, 1000}, + Help: "number of errors that occurred during the async inspection of publish messages on a single RPC, regardless pof the result", + }) + + gc.publishMessageInvalidSenderCountHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_publish_message_invalid_sender_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of invalid senders observed during the async inspection of publish messages on a single RPC, regardless of the result", + }) + + gc.publishMessageInvalidTopicIdHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_publish_message_invalid_topic_id_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of invalid topic ids observed during the async inspection of publish messages on a single RPC, regardless of the result", + }) + + gc.publishMessageInvalidSubscriptionsHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "rpc_inspection_publish_message_invalid_subscriptions_count", + Buckets: []float64{1, 100, 1000}, + Help: "number of invalid subscriptions observed during the async inspection of publish messages on a single RPC, regardless of the result", + }) + + gc.publishMessageInspectionErrExceedThresholdCount = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemGossip, + Name: gc.prefix + "publish_message_inspection_err_exceed_threshold_total", + Help: "number of rpcs fail on inspection due to published message inspection errors exceeding the threshold", }) return gc @@ -188,3 +378,121 @@ func (c *GossipSubRpcValidationInspectorMetrics) OnIncomingRpcReceived(iHaveCoun c.receivedIWantMsgCount.Add(float64(iWantCount)) c.receivedIHaveMsgCount.Add(float64(iHaveCount)) } + +// OnIWantMessagesInspected tracks the number of duplicate and cache miss message ids received by the node on iWant messages at the end of the async inspection iWants +// across one RPC, regardless of the result of the inspection. +// +// duplicateCount: the total number of duplicate message ids received by the node on the iWant messages at the end of the async inspection of the RPC. +// cacheMissCount: the total number of cache miss message ids received by the node on the iWant message at the end of the async inspection of the RPC. +func (c *GossipSubRpcValidationInspectorMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + c.iWantDuplicateMessageIdHistogram.Observe(float64(duplicateCount)) + c.iWantCacheMissHistogram.Observe(float64(cacheMissCount)) +} + +// OnIWantDuplicateMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total number of duplicate message ids +// received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + c.iWantDuplicateMessageIdExceedThresholdCount.Inc() +} + +// OnIWantCacheMissMessageIdsExceedThreshold tracks the number of times that async inspection of iWant messages failed due to the total +// number of cache miss message ids received by the node on the iWant messages of a single RPC exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + c.iWantCacheMissMessageIdExceedThresholdCount.Inc() +} + +// OnIHaveMessagesInspected is called at the end of the async inspection of iHave messages of a single RPC, regardless of the result of the inspection. +// It tracks the number of duplicate topic ids and duplicate message ids received by the node on the iHave messages of that single RPC at the end of the async inspection iHaves. +// Args: +// +// duplicateTopicIds: the total number of duplicate topic ids received by the node on the iHave messages at the end of the async inspection of the RPC. +// duplicateMessageIds: the number of duplicate message ids received by the node on the iHave messages at the end of the async inspection of the RPC. +func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + c.iHaveDuplicateTopicIdHistogram.Observe(float64(duplicateTopicIds)) + c.iHaveDuplicateMessageIdHistogram.Observe(float64(duplicateMessageIds)) +} + +// OnIHaveDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate topic ids +// received by the node on the iHave messages of that RPC exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + c.iHaveDuplicateTopicIdExceedThresholdCount.Inc() +} + +// OnIHaveDuplicateMessageIdsExceedThreshold tracks the number of times that the async inspection of iHave messages of a single RPC failed due to the total number of duplicate message ids +// received by the node on an iHave message exceeding the threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + c.iHaveDuplicateMessageIdExceedThresholdCount.Inc() +} + +// OnInvalidTopicIdDetectedForControlMessage tracks the number of times that the async inspection of a control message type on a single RPC failed due to an invalid topic id. +// Args: +// - messageType: the type of the control message that was truncated. +func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + c.ctrlMsgInvalidTopicIdCount.WithLabelValues(messageType.String()).Inc() +} + +// OnActiveClusterIDsNotSetErr tracks the number of times that the async inspection of a control message type on a single RPC failed due to active cluster ids not set inspection failure. +// This is not causing a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnActiveClusterIDsNotSetErr() { + c.errActiveClusterIdsNotSetCount.Inc() +} + +// OnUnstakedPeerInspectionFailed tracks the number of times that the async inspection of a control message type on a single RPC failed due to unstaked peer inspection failure. +// This is not causing a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnUnstakedPeerInspectionFailed() { + c.errUnstakedPeerInspectionFailedCount.Inc() +} + +// OnInvalidControlMessageNotificationSent tracks the number of times that the async inspection of a control message failed and resulted in dissemination of an invalid control message was sent (i.e., a +// misbehavior report). +func (c *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageNotificationSent() { + c.invalidControlMessageNotificationSentCount.Inc() +} + +// OnPruneDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of prune messages for an RPC failed due to the number of duplicate topic ids +// received by the node on prune messages of the same RPC excesses threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + c.pruneDuplicateTopicIdsExceedThresholdCount.Inc() +} + +// OnPruneMessageInspected is called at the end of the async inspection of prune messages of the RPC, regardless of the result of the inspection. +// Args: +// +// duplicateTopicIds: the number of duplicate topic ids received by the node on the prune messages of the RPC at the end of the async inspection prunes. +func (c *GossipSubRpcValidationInspectorMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + c.pruneDuplicateTopicIdsHistogram.Observe(float64(duplicateTopicIds)) +} + +// OnGraftDuplicateTopicIdsExceedThreshold tracks the number of times that the async inspection of a graft message failed due to the number of duplicate topic ids. +// received by the node on graft messages of the same rpc excesses threshold, which results in a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + c.graftDuplicateTopicIdsExceedThresholdCount.Inc() +} + +// OnGraftMessageInspected is called at the end of the async inspection of graft messages of a single RPC, regardless of the result of the inspection. +// Args: +// +// duplicateTopicIds: the number of duplicate topic ids received by the node on the graft messages at the end of the async inspection of a single RPC. +func (c *GossipSubRpcValidationInspectorMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + c.graftDuplicateTopicIdsHistogram.Observe(float64(duplicateTopicIds)) +} + +// OnPublishMessageInspected is called at the end of the async inspection of publish messages of a single RPC, regardless of the result of the inspection. +// It tracks the total number of errors detected during the async inspection of the rpc together with their individual breakdown. +// Args: +// - errCount: the number of errors that occurred during the async inspection of publish messages. +// - invalidTopicIdsCount: the number of times that an invalid topic id was detected during the async inspection of publish messages. +// - invalidSubscriptionsCount: the number of times that an invalid subscription was detected during the async inspection of publish messages. +// - invalidSendersCount: the number of times that an invalid sender was detected during the async inspection of publish messages. +func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + c.publishMessageInspectedErrHistogram.Observe(float64(totalErrCount)) + c.publishMessageInvalidSenderCountHistogram.Observe(float64(invalidSendersCount)) + c.publishMessageInvalidSubscriptionsHistogram.Observe(float64(invalidSubscriptionsCount)) + c.publishMessageInvalidTopicIdHistogram.Observe(float64(invalidTopicIdsCount)) +} + +// OnPublishMessagesInspectionErrorExceedsThreshold tracks the number of times that async inspection of publish messages failed due to the number of errors exceeding the threshold. +// Note that it causes a misbehaviour report. +func (c *GossipSubRpcValidationInspectorMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + c.publishMessageInspectionErrExceedThresholdCount.Inc() +} diff --git a/module/metrics/noop.go b/module/metrics/noop.go index b2d2f8d08aa..14ce9bb3994 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -323,8 +323,26 @@ func (nc *NoopCollector) OnControlMessagesTruncated(messageType p2pmsg.ControlMe } func (nc *NoopCollector) OnIncomingRpcReceived(iHaveCount, iWantCount, graftCount, pruneCount, msgCount int) { } -func (nc *NoopCollector) AsyncProcessingStarted() {} -func (nc *NoopCollector) AsyncProcessingFinished(time.Duration) {} +func (nc *NoopCollector) AsyncProcessingStarted() {} +func (nc *NoopCollector) AsyncProcessingFinished(time.Duration) {} +func (nc *NoopCollector) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) {} +func (nc *NoopCollector) OnIWantDuplicateMessageIdsExceedThreshold() {} +func (nc *NoopCollector) OnIWantCacheMissMessageIdsExceedThreshold() {} +func (nc *NoopCollector) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) {} +func (nc *NoopCollector) OnIHaveDuplicateTopicIdsExceedThreshold() {} +func (nc *NoopCollector) OnIHaveDuplicateMessageIdsExceedThreshold() {} +func (nc *NoopCollector) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { +} +func (nc *NoopCollector) OnActiveClusterIDsNotSetErr() {} +func (nc *NoopCollector) OnUnstakedPeerInspectionFailed() {} +func (nc *NoopCollector) OnInvalidControlMessageNotificationSent() {} +func (nc *NoopCollector) OnPublishMessagesInspectionErrorExceedsThreshold() {} +func (nc *NoopCollector) OnPruneDuplicateTopicIdsExceedThreshold() {} +func (nc *NoopCollector) OnPruneMessageInspected(duplicateTopicIds int) {} +func (nc *NoopCollector) OnGraftDuplicateTopicIdsExceedThreshold() {} +func (nc *NoopCollector) OnGraftMessageInspected(duplicateTopicIds int) {} +func (nc *NoopCollector) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { +} func (nc *NoopCollector) OnMisbehaviorReported(string, string) {} func (nc *NoopCollector) OnViolationReportSkipped() {} diff --git a/module/mock/gossip_sub_metrics.go b/module/mock/gossip_sub_metrics.go index f0fbdec5cfd..f7e057ea5ba 100644 --- a/module/mock/gossip_sub_metrics.go +++ b/module/mock/gossip_sub_metrics.go @@ -26,6 +26,11 @@ func (_m *GossipSubMetrics) AsyncProcessingStarted() { _m.Called() } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *GossipSubMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnAppSpecificScoreUpdated provides a mock function with given fields: _a0 func (_m *GossipSubMetrics) OnAppSpecificScoreUpdated(_a0 float64) { _m.Called(_a0) @@ -46,41 +51,91 @@ func (_m *GossipSubMetrics) OnFirstMessageDeliveredUpdated(_a0 channels.Topic, _ _m.Called(_a0, _a1) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *GossipSubMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *GossipSubMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + // OnIPColocationFactorUpdated provides a mock function with given fields: _a0 func (_m *GossipSubMetrics) OnIPColocationFactorUpdated(_a0 float64) { _m.Called(_a0) } +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *GossipSubMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *GossipSubMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *GossipSubMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: +func (_m *GossipSubMetrics) OnInvalidControlMessageNotificationSent() { + _m.Called() +} + // OnInvalidMessageDeliveredUpdated provides a mock function with given fields: _a0, _a1 func (_m *GossipSubMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _a1 float64) { _m.Called(_a0, _a1) } +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *GossipSubMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + // OnLocalMeshSizeUpdated provides a mock function with given fields: topic, size func (_m *GossipSubMetrics) OnLocalMeshSizeUpdated(topic string, size int) { _m.Called(topic, size) @@ -156,6 +211,26 @@ func (_m *GossipSubMetrics) OnPeerThrottled() { _m.Called() } +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount +func (_m *GossipSubMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *GossipSubMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + // OnRpcReceived provides a mock function with given fields: msgCount, iHaveCount, iWantCount, graftCount, pruneCount func (_m *GossipSubMetrics) OnRpcReceived(msgCount int, iHaveCount int, iWantCount int, graftCount int, pruneCount int) { _m.Called(msgCount, iHaveCount, iWantCount, graftCount, pruneCount) @@ -176,6 +251,11 @@ func (_m *GossipSubMetrics) OnUndeliveredMessage() { _m.Called() } +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *GossipSubMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + // SetWarningStateCount provides a mock function with given fields: _a0 func (_m *GossipSubMetrics) SetWarningStateCount(_a0 uint) { _m.Called(_a0) diff --git a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go index 0a10b099844..84eef02f7ea 100644 --- a/module/mock/gossip_sub_rpc_validation_inspector_metrics.go +++ b/module/mock/gossip_sub_rpc_validation_inspector_metrics.go @@ -25,36 +25,116 @@ func (_m *GossipSubRpcValidationInspectorMetrics) AsyncProcessingStarted() { _m.Called() } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnControlMessagesTruncated provides a mock function with given fields: messageType, diff func (_m *GossipSubRpcValidationInspectorMetrics) OnControlMessagesTruncated(messageType p2pmsg.ControlMessageType, diff int) { _m.Called(messageType, diff) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubRpcValidationInspectorMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *GossipSubRpcValidationInspectorMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *GossipSubRpcValidationInspectorMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *GossipSubRpcValidationInspectorMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidControlMessageNotificationSent() { + _m.Called() +} + +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *GossipSubRpcValidationInspectorMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *GossipSubRpcValidationInspectorMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount +func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *GossipSubRpcValidationInspectorMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + type mockConstructorTestingTNewGossipSubRpcValidationInspectorMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/lib_p2_p_metrics.go b/module/mock/lib_p2_p_metrics.go index d555414cd6a..f91d247d6bf 100644 --- a/module/mock/lib_p2_p_metrics.go +++ b/module/mock/lib_p2_p_metrics.go @@ -112,6 +112,11 @@ func (_m *LibP2PMetrics) InboundConnections(connectionCount uint) { _m.Called(connectionCount) } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *LibP2PMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnAppSpecificScoreUpdated provides a mock function with given fields: _a0 func (_m *LibP2PMetrics) OnAppSpecificScoreUpdated(_a0 float64) { _m.Called(_a0) @@ -167,41 +172,91 @@ func (_m *LibP2PMetrics) OnFirstMessageDeliveredUpdated(_a0 channels.Topic, _a1 _m.Called(_a0, _a1) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *LibP2PMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *LibP2PMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *LibP2PMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *LibP2PMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + // OnIPColocationFactorUpdated provides a mock function with given fields: _a0 func (_m *LibP2PMetrics) OnIPColocationFactorUpdated(_a0 float64) { _m.Called(_a0) } +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *LibP2PMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *LibP2PMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *LibP2PMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *LibP2PMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: +func (_m *LibP2PMetrics) OnInvalidControlMessageNotificationSent() { + _m.Called() +} + // OnInvalidMessageDeliveredUpdated provides a mock function with given fields: _a0, _a1 func (_m *LibP2PMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _a1 float64) { _m.Called(_a0, _a1) } +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *LibP2PMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + // OnLocalMeshSizeUpdated provides a mock function with given fields: topic, size func (_m *LibP2PMetrics) OnLocalMeshSizeUpdated(topic string, size int) { _m.Called(topic, size) @@ -287,6 +342,26 @@ func (_m *LibP2PMetrics) OnPeerThrottled() { _m.Called() } +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *LibP2PMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount +func (_m *LibP2PMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *LibP2PMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + // OnRpcReceived provides a mock function with given fields: msgCount, iHaveCount, iWantCount, graftCount, pruneCount func (_m *LibP2PMetrics) OnRpcReceived(msgCount int, iHaveCount int, iWantCount int, graftCount int, pruneCount int) { _m.Called(msgCount, iHaveCount, iWantCount, graftCount, pruneCount) @@ -332,6 +407,11 @@ func (_m *LibP2PMetrics) OnUndeliveredMessage() { _m.Called() } +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *LibP2PMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + // OutboundConnections provides a mock function with given fields: connectionCount func (_m *LibP2PMetrics) OutboundConnections(connectionCount uint) { _m.Called(connectionCount) diff --git a/module/mock/network_metrics.go b/module/mock/network_metrics.go index cc941222bc8..e86d63fb03a 100644 --- a/module/mock/network_metrics.go +++ b/module/mock/network_metrics.go @@ -142,6 +142,11 @@ func (_m *NetworkMetrics) MessageRemoved(priority int) { _m.Called(priority) } +// OnActiveClusterIDsNotSetErr provides a mock function with given fields: +func (_m *NetworkMetrics) OnActiveClusterIDsNotSetErr() { + _m.Called() +} + // OnAppSpecificScoreUpdated provides a mock function with given fields: _a0 func (_m *NetworkMetrics) OnAppSpecificScoreUpdated(_a0 float64) { _m.Called(_a0) @@ -197,41 +202,91 @@ func (_m *NetworkMetrics) OnFirstMessageDeliveredUpdated(_a0 channels.Topic, _a1 _m.Called(_a0, _a1) } +// OnGraftDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnGraftDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnGraftMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *NetworkMetrics) OnGraftMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + // OnIHaveControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *NetworkMetrics) OnIHaveControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIHaveDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIHaveDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + +// OnIHaveDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIHaveDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + // OnIHaveMessageIDsReceived provides a mock function with given fields: channel, msgIdCount func (_m *NetworkMetrics) OnIHaveMessageIDsReceived(channel string, msgIdCount int) { _m.Called(channel, msgIdCount) } +// OnIHaveMessagesInspected provides a mock function with given fields: duplicateTopicIds, duplicateMessageIds +func (_m *NetworkMetrics) OnIHaveMessagesInspected(duplicateTopicIds int, duplicateMessageIds int) { + _m.Called(duplicateTopicIds, duplicateMessageIds) +} + // OnIPColocationFactorUpdated provides a mock function with given fields: _a0 func (_m *NetworkMetrics) OnIPColocationFactorUpdated(_a0 float64) { _m.Called(_a0) } +// OnIWantCacheMissMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIWantCacheMissMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantControlMessageIdsTruncated provides a mock function with given fields: diff func (_m *NetworkMetrics) OnIWantControlMessageIdsTruncated(diff int) { _m.Called(diff) } +// OnIWantDuplicateMessageIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnIWantDuplicateMessageIdsExceedThreshold() { + _m.Called() +} + // OnIWantMessageIDsReceived provides a mock function with given fields: msgIdCount func (_m *NetworkMetrics) OnIWantMessageIDsReceived(msgIdCount int) { _m.Called(msgIdCount) } +// OnIWantMessagesInspected provides a mock function with given fields: duplicateCount, cacheMissCount +func (_m *NetworkMetrics) OnIWantMessagesInspected(duplicateCount int, cacheMissCount int) { + _m.Called(duplicateCount, cacheMissCount) +} + // OnIncomingRpcReceived provides a mock function with given fields: iHaveCount, iWantCount, graftCount, pruneCount, msgCount func (_m *NetworkMetrics) OnIncomingRpcReceived(iHaveCount int, iWantCount int, graftCount int, pruneCount int, msgCount int) { _m.Called(iHaveCount, iWantCount, graftCount, pruneCount, msgCount) } +// OnInvalidControlMessageNotificationSent provides a mock function with given fields: +func (_m *NetworkMetrics) OnInvalidControlMessageNotificationSent() { + _m.Called() +} + // OnInvalidMessageDeliveredUpdated provides a mock function with given fields: _a0, _a1 func (_m *NetworkMetrics) OnInvalidMessageDeliveredUpdated(_a0 channels.Topic, _a1 float64) { _m.Called(_a0, _a1) } +// OnInvalidTopicIdDetectedForControlMessage provides a mock function with given fields: messageType +func (_m *NetworkMetrics) OnInvalidTopicIdDetectedForControlMessage(messageType p2pmsg.ControlMessageType) { + _m.Called(messageType) +} + // OnLocalMeshSizeUpdated provides a mock function with given fields: topic, size func (_m *NetworkMetrics) OnLocalMeshSizeUpdated(topic string, size int) { _m.Called(topic, size) @@ -322,6 +377,26 @@ func (_m *NetworkMetrics) OnPeerThrottled() { _m.Called() } +// OnPruneDuplicateTopicIdsExceedThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnPruneDuplicateTopicIdsExceedThreshold() { + _m.Called() +} + +// OnPruneMessageInspected provides a mock function with given fields: duplicateTopicIds +func (_m *NetworkMetrics) OnPruneMessageInspected(duplicateTopicIds int) { + _m.Called(duplicateTopicIds) +} + +// OnPublishMessageInspected provides a mock function with given fields: totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount +func (_m *NetworkMetrics) OnPublishMessageInspected(totalErrCount int, invalidTopicIdsCount int, invalidSubscriptionsCount int, invalidSendersCount int) { + _m.Called(totalErrCount, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) +} + +// OnPublishMessagesInspectionErrorExceedsThreshold provides a mock function with given fields: +func (_m *NetworkMetrics) OnPublishMessagesInspectionErrorExceedsThreshold() { + _m.Called() +} + // OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason func (_m *NetworkMetrics) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { _m.Called(pid, role, msgType, topic, reason) @@ -377,6 +452,11 @@ func (_m *NetworkMetrics) OnUndeliveredMessage() { _m.Called() } +// OnUnstakedPeerInspectionFailed provides a mock function with given fields: +func (_m *NetworkMetrics) OnUnstakedPeerInspectionFailed() { + _m.Called() +} + // OnViolationReportSkipped provides a mock function with given fields: func (_m *NetworkMetrics) OnViolationReportSkipped() { _m.Called() diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 7c8f8ca5d90..3d37c078bc1 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -83,22 +83,24 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerQueueCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerNumOfWorkersKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.NumberOfWorkersKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.QueueSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.NumberOfWorkersKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.QueueSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.TrackerCacheSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.TrackerCacheDecayKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.HardThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.NotificationCacheSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneMessageMaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageCountThreshold), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageIdCountThreshold), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MessageCountThreshold), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageCountThreshold), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageIdCountThreshold), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissCheckSizeKey), BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageMaxSampleSizeKey), - BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageErrorThresholdKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MaxSampleSizeKey), + BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.CacheSizeKey), @@ -230,11 +232,11 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcTracerKey, p2pconfig.RPCSentTrackerNumOfWorkersKey), config.GossipSub.RpcTracer.RpcSentTrackerNumOfWorkers, "number of workers for the rpc sent tracker worker pool.") // gossipsub RPC control message validation limits used for validation configuration and rate limiting - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.NumberOfWorkersKey), - config.GossipSub.RpcInspector.Validation.NumberOfWorkers, + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.NumberOfWorkersKey), + config.GossipSub.RpcInspector.Validation.InspectionQueue.NumberOfWorkers, "number of gossipsub RPC control message validation inspector component workers") - flags.Uint32(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.QueueSizeKey), - config.GossipSub.RpcInspector.Validation.QueueSize, + flags.Uint32(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.InspectionQueueConfigKey, p2pconfig.QueueSizeKey), + config.GossipSub.RpcInspector.Validation.InspectionQueue.Size, "queue size for gossipsub RPC validation inspector events worker pool queue.") flags.Uint32(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.ClusterPrefixedMessageConfigKey, p2pconfig.TrackerCacheSizeKey), config.GossipSub.RpcInspector.Validation.ClusterPrefixedMessage.ControlMsgsReceivedCacheSize, @@ -262,37 +264,42 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { config.AlspConfig.SyncEngine.RangeRequestBaseProb, "base probability of creating a misbehavior report for a range request message") flags.Float32(alspSyncEngineSyncRequestProb, config.AlspConfig.SyncEngine.SyncRequestProb, "probability of creating a misbehavior report for a sync request message") - - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxSampleSize, - "max number of ihaves to sample when performing validation") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IHave.MaxMessageIDSampleSize, - "max number of message ids to sample when performing validation per ihave") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneMessageMaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.GraftPruneMessageMaxSampleSize, - "max number of control messages to sample when performing validation on GRAFT and PRUNE message types") - flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.MaxSampleSize, - "max number of iwants to sample when performing validation") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MaxMessageIDSampleSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.MaxMessageIDSampleSize, - "max number of message ids to sample when performing validation per iwant") - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageCountThreshold), + config.GossipSub.RpcInspector.Validation.IHave.MessageCountThreshold, + "threshold for the number of ihave control messages to accept on a single RPC message, if exceeded the RPC message will be sampled and truncated") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.MessageIdCountThreshold), + config.GossipSub.RpcInspector.Validation.IHave.MessageIdCountThreshold, + "threshold for the number of message ids on a single ihave control message to accept, if exceeded the RPC message ids will be sampled and truncated") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold, + "the max allowed duplicate topic IDs across all ihave control messages in a single RPC message, if exceeded a misbehavior report will be created") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IHaveConfigKey, p2pconfig.DuplicateMessageIdThresholdKey), + config.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold, + "the max allowed duplicate message IDs in a single ihave control message, if exceeded a misbehavior report will be created") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.MessageCountThreshold), + config.GossipSub.RpcInspector.Validation.GraftPrune.MessageCountThreshold, + "threshold for the number of graft or prune control messages to accept on a single RPC message, if exceeded the RPC message will be sampled and truncated") + flags.Uint(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageCountThreshold), + config.GossipSub.RpcInspector.Validation.IWant.MessageCountThreshold, + "threshold for the number of iwant control messages to accept on a single RPC message, if exceeded the RPC message will be sampled and truncated") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.MessageIdCountThreshold), + config.GossipSub.RpcInspector.Validation.IWant.MessageIdCountThreshold, + "threshold for the number of message ids on a single iwant control message to accept, if exceeded the RPC message ids will be sampled and truncated") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissThresholdKey), config.GossipSub.RpcInspector.Validation.IWant.CacheMissThreshold, - "max number of iwants to sample when performing validation") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.CacheMissCheckSizeKey), - config.GossipSub.RpcInspector.Validation.IWant.CacheMissCheckSize, - "the iWants size at which message id cache misses will be checked") - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), - config.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIDThreshold, - "max allowed duplicate message IDs in a single iWant control message") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageMaxSampleSizeKey), - config.GossipSub.RpcInspector.Validation.MessageMaxSampleSize, - "the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated") - flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.MessageErrorThresholdKey), - config.GossipSub.RpcInspector.Validation.MessageErrorThreshold, - "the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value") + "max number of cache misses (untracked) allowed in a single iWant control message, if exceeded a misbehavior report will be created") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.IWantConfigKey, p2pconfig.DuplicateMsgIDThresholdKey), + config.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIdThreshold, + "max allowed duplicate message IDs in a single iWant control message, if exceeded a misbehavior report will be created") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MaxSampleSizeKey), + config.GossipSub.RpcInspector.Validation.PublishMessages.MaxSampleSize, + "the max sample size for async validation of publish messages, if exceeded the message will be sampled for inspection, but is not truncated") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.PublishMessagesConfigKey, p2pconfig.MessageErrorThresholdKey), + config.GossipSub.RpcInspector.Validation.PublishMessages.ErrorThreshold, + "the max number of errors allowed in a (sampled) set of publish messages on a single rpc, if exceeded a misbehavior report will be created") + flags.Int(BuildFlagName(gossipsubKey, p2pconfig.RpcInspectorKey, p2pconfig.ValidationConfigKey, p2pconfig.GraftPruneKey, p2pconfig.DuplicateTopicIdThresholdKey), + config.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold, + "the max allowed duplicate topic IDs across all graft or prune control messages in a single RPC message, if exceeded a misbehavior report will be created") flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.SubscriptionProviderKey, p2pconfig.UpdateIntervalKey), config.GossipSub.SubscriptionProvider.UpdateInterval, "interval for updating the list of subscribed topics for all peers in the gossipsub, recommended value is a few minutes") @@ -382,7 +389,13 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.PeerScoringKey, p2pconfig.ProtocolKey, p2pconfig.AppSpecificKey, p2pconfig.UnknownIdentityKey, p2pconfig.PenaltyKey), config.GossipSub.ScoringParameters.PeerScoring.Protocol.AppSpecificScore.UnknownIdentityPenalty, "the penalty for unknown identity. It is applied to the peer's score when the peer is not in the identity list") - flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.PeerScoringKey, p2pconfig.ProtocolKey, p2pconfig.AppSpecificKey, p2pconfig.InvalidSubscriptionKey, p2pconfig.PenaltyKey), + flags.Float64(BuildFlagName(gossipsubKey, + p2pconfig.ScoreParamsKey, + p2pconfig.PeerScoringKey, + p2pconfig.ProtocolKey, + p2pconfig.AppSpecificKey, + p2pconfig.InvalidSubscriptionKey, + p2pconfig.PenaltyKey), config.GossipSub.ScoringParameters.PeerScoring.Protocol.AppSpecificScore.InvalidSubscriptionPenalty, "the penalty for invalid subscription. It is applied to the peer's score when the peer subscribes to a topic that it is not authorized to subscribe to") flags.Float64(BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.PeerScoringKey, p2pconfig.ProtocolKey, p2pconfig.AppSpecificKey, p2pconfig.MaxAppSpecificKey, p2pconfig.RewardKey), diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index 829cb33ca61..fec58ce5fe1 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -17,67 +17,137 @@ type RpcInspectorParameters struct { // RpcValidationInspectorParameters keys. const ( - ClusterPrefixedMessageConfigKey = "cluster-prefixed-messages" - IWantConfigKey = "iwant" - IHaveConfigKey = "ihave" - QueueSizeKey = "queue-size" - GraftPruneMessageMaxSampleSizeKey = "graft-and-prune-message-max-sample-size" - MessageMaxSampleSizeKey = "message-max-sample-size" - MessageErrorThresholdKey = "error-threshold" + ClusterPrefixedMessageConfigKey = "cluster-prefixed-messages" + IWantConfigKey = "iwant" + IHaveConfigKey = "ihave" + GraftPruneKey = "graft-and-prune" + PublishMessagesConfigKey = "publish-messages" + InspectionQueueConfigKey = "inspection-queue" ) // RpcValidationInspector validation limits used for gossipsub RPC control message inspection. type RpcValidationInspector struct { ClusterPrefixedMessage ClusterPrefixedMessageInspectionParameters `mapstructure:"cluster-prefixed-messages"` - IWant IWantRPCInspectionParameters `mapstructure:"iwant"` + IWant IWantRpcInspectionParameters `mapstructure:"iwant"` IHave IHaveRpcInspectionParameters `mapstructure:"ihave"` + GraftPrune GraftPruneRpcInspectionParameters `mapstructure:"graft-and-prune"` + PublishMessages PublishMessageInspectionParameters `mapstructure:"publish-messages"` + InspectionQueue InspectionQueueParameters `mapstructure:"inspection-queue"` +} + +const ( + QueueSizeKey = "queue-size" +) + +// InspectionQueueParameters contains the "numerical values" for the control message validation inspector. +// Incoming GossipSub RPCs are queued for async inspection by a worker pool. This worker pool is configured +// by the parameters in this struct. +// Each RPC has a number of "publish messages" accompanied by control messages. +type InspectionQueueParameters struct { // NumberOfWorkers number of worker pool workers. NumberOfWorkers int `validate:"gte=1" mapstructure:"workers"` - // QueueSize size of the queue used by worker pool for the control message validation inspector. - QueueSize uint32 `validate:"gte=100" mapstructure:"queue-size"` - // GraftPruneMessageMaxSampleSize the max sample size used for control message validation of GRAFT and PRUNE. If the total number of control messages (GRAFT or PRUNE) - // exceeds this max sample size then the respective message will be truncated to this value before being processed. - GraftPruneMessageMaxSampleSize int `validate:"gte=1000" mapstructure:"graft-and-prune-message-max-sample-size"` - // RPCMessageMaxSampleSize the max sample size used for RPC message validation. If the total number of RPC messages exceeds this value a sample will be taken but messages will not be truncated. - MessageMaxSampleSize int `validate:"gte=1000" mapstructure:"message-max-sample-size"` - // RPCMessageErrorThreshold the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value. - MessageErrorThreshold int `validate:"gte=500" mapstructure:"error-threshold"` + // Size size of the queue used by worker pool for the control message validation inspector. + Size uint32 `validate:"gte=100" mapstructure:"queue-size"` +} + +const ( + MaxSampleSizeKey = "max-sample-size" + MessageErrorThresholdKey = "error-threshold" +) + +// PublishMessageInspectionParameters contains the "numerical values" for the publish control message inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of these publish messages. +type PublishMessageInspectionParameters struct { + // MaxSampleSize is the maximum number of messages in a single RPC message that are randomly sampled for async inspection. + // When the size of a single RPC message exceeds this threshold, a random sample is taken for inspection, but the RPC message is not truncated. + MaxSampleSize int `validate:"gte=0" mapstructure:"max-sample-size"` + // ErrorThreshold the threshold at which an error will be returned if the number of invalid RPC messages exceeds this value. + ErrorThreshold int `validate:"gte=0" mapstructure:"error-threshold"` +} + +// GraftPruneRpcInspectionParameters contains the "numerical values" for the graft and prune control message inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of these graft and prune control messages. +type GraftPruneRpcInspectionParameters struct { + // MessageCountThreshold is the maximum number of GRAFT or PRUNE messages in a single RPC message. + // When the total number of GRAFT or PRUNE messages in a single RPC message exceeds this threshold, + // a random sample of GRAFT or PRUNE messages will be taken and the RPC message will be truncated to this sample size. + MessageCountThreshold int `validate:"gte=0" mapstructure:"message-count-threshold"` + + // DuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in a single GRAFT or PRUNE message under inspection. + // Ideally, a GRAFT or PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. + // When the total number of duplicate topic ids in a single GRAFT or PRUNE message exceeds this threshold, the inspection of message will fail. + DuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` } const ( - MaxSampleSizeKey = "max-sample-size" - MaxMessageIDSampleSizeKey = "max-message-id-sample-size" + MessageCountThreshold = "message-count-threshold" + MessageIdCountThreshold = "message-id-count-threshold" CacheMissThresholdKey = "cache-miss-threshold" - CacheMissCheckSizeKey = "cache-miss-check-size" DuplicateMsgIDThresholdKey = "duplicate-message-id-threshold" ) -// IWantRPCInspectionParameters contains the "numerical values" for the iwant rpc control message inspection. -type IWantRPCInspectionParameters struct { - // MaxSampleSize max inspection sample size to use. If the total number of iWant control messages - // exceeds this max sample size then the respective message will be truncated before being processed. - MaxSampleSize uint `validate:"gt=0" mapstructure:"max-sample-size"` - // MaxMessageIDSampleSize max inspection sample size to use for iWant message ids. Each iWant message includes a list of message ids - // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` - // CacheMissThreshold the threshold of missing corresponding iHave messages for iWant messages received before an invalid control message notification is disseminated. - // If the cache miss threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. - CacheMissThreshold float64 `validate:"gt=0" mapstructure:"cache-miss-threshold"` - // CacheMissCheckSize the iWants size at which message id cache misses will be checked. - CacheMissCheckSize int `validate:"gt=0" mapstructure:"cache-miss-check-size"` - // DuplicateMsgIDThreshold maximum allowed duplicate message IDs in a single iWant control message. - // If the duplicate message threshold is exceeded an invalid control message notification is disseminated and the sender will be penalized. - DuplicateMsgIDThreshold float64 `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` +// IWantRpcInspectionParameters contains the "numerical values" for iwant rpc control inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of the iwant control messages. +type IWantRpcInspectionParameters struct { + // MessageCountThreshold is the maximum allowed number of iWant messages in a single RPC message. + // Each iWant message represents the list of message ids. When the total number of iWant messages + // in a single RPC message exceeds this threshold, a random sample of iWant messages will be taken and the RPC message will be truncated to this sample size. + // The sample size is equal to the configured MessageCountThreshold. + MessageCountThreshold uint `validate:"gt=0" mapstructure:"message-count-threshold"` + // MessageIdCountThreshold is the maximum allowed number of message ids in a single iWant message. + // Each iWant message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + // that can be included in a single iWant message. When the total number of message ids in a single iWant message exceeds this threshold, + // a random sample of message ids will be taken and the iWant message will be truncated to this sample size. + // The sample size is equal to the configured MessageIdCountThreshold. + MessageIdCountThreshold int `validate:"gte=0" mapstructure:"message-id-count-threshold"` + // CacheMissThreshold is the threshold of tolerance for the total cache misses in all iWant messages in a single RPC message. + // When the total number of cache misses in all iWant messages in a single RPC message exceeds this threshold, the inspection of message will fail. + // An iWant message is considered a cache miss if it contains a message id that is not present in the local cache for iHave messages, i.e., the node + // does not have a record of an iHave message for this message id. + // When the total number of cache misses in all iWant messages in a single RPC message exceeds this threshold, the inspection of message will fail, and + // a single misbehavior notification will be reported. + CacheMissThreshold int `validate:"gt=0" mapstructure:"cache-miss-threshold"` + // DuplicateMsgIdThreshold is the maximum allowed number of duplicate message ids in a all iWant messages in a single RPC message. + // Each iWant message represents the list of message ids, and this parameter controls the maximum number of duplicate message ids + // that can be included in all iWant messages in a single RPC message. When the total number of duplicate message ids in a single iWant message exceeds this threshold, + // a single misbehavior notification will be reported, and the inspection of message will fail. + DuplicateMsgIdThreshold int `validate:"gt=0" mapstructure:"duplicate-message-id-threshold"` } +const ( + DuplicateTopicIdThresholdKey = "duplicate-topic-id-threshold" + DuplicateMessageIdThresholdKey = "duplicate-message-id-threshold" +) + // IHaveRpcInspectionParameters contains the "numerical values" for ihave rpc control inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits +// for the inspection of the ihave control messages. type IHaveRpcInspectionParameters struct { - // MaxSampleSize max inspection sample size to use. If the number of ihave messages exceeds this configured value - // the control message ihaves will be truncated to the max sample size. This sample is randomly selected. - MaxSampleSize int `validate:"gte=1000" mapstructure:"max-sample-size"` - // MaxMessageIDSampleSize max inspection sample size to use for iHave message ids. Each ihave message includes a list of message ids - // each, if the size of this list exceeds the configured max message id sample size the list of message ids will be truncated. - MaxMessageIDSampleSize int `validate:"gte=1000" mapstructure:"max-message-id-sample-size"` + // MessageCountThreshold is the maximum allowed number of iHave messages in a single RPC message. + // Each iHave message represents the list of message ids for a specific topic. When the total number of iHave messages + // in a single RPC message exceeds this threshold, a random sample of iHave messages will be taken and the RPC message will be truncated to this sample size. + // The sample size is equal to the configured MessageCountThreshold. + MessageCountThreshold int `validate:"gte=0" mapstructure:"message-count-threshold"` + // MessageIdCountThreshold is the maximum allowed number of message ids in a single iHave message. + // Each iHave message represents the list of message ids for a specific topic, and this parameter controls the maximum number of message ids + // that can be included in a single iHave message. When the total number of message ids in a single iHave message exceeds this threshold, + // a random sample of message ids will be taken and the iHave message will be truncated to this sample size. + // The sample size is equal to the configured MessageIdCountThreshold. + MessageIdCountThreshold int `validate:"gte=0" mapstructure:"message-id-count-threshold"` + + // DuplicateTopicIdThreshold is the tolerance threshold for having duplicate topics in an iHave message under inspection. + // When the total number of duplicate topic ids in a single iHave message exceeds this threshold, the inspection of message will fail. + // Note that a topic ID is counted as a duplicate only if it is repeated more than DuplicateTopicIdThreshold times. + DuplicateTopicIdThreshold int `validate:"gte=0" mapstructure:"duplicate-topic-id-threshold"` + + // DuplicateMessageIdThreshold is the threshold of tolerance for having duplicate message IDs in a single iHave message under inspection. + // When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + // Ideally, an iHave message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once + // within the same iHave message. When the total number of duplicate message ids in a single iHave message exceeds this threshold, the inspection of message will fail. + DuplicateMessageIdThreshold int `validate:"gte=0" mapstructure:"duplicate-message-id-threshold"` } const ( @@ -87,6 +157,9 @@ const ( ) // ClusterPrefixedMessageInspectionParameters contains the "numerical values" for cluster prefixed control message inspection. +// Each RPC has a number of "publish messages" accompanied by control messages. This struct contains the limits for the inspection +// of messages (publish messages and control messages) that belongs to cluster prefixed topics. +// Cluster-prefixed topics are topics that are prefixed with the cluster ID of the node that published the message. type ClusterPrefixedMessageInspectionParameters struct { // HardThreshold the upper bound on the amount of cluster prefixed control messages that will be processed // before a node starts to get penalized. This allows LN nodes to process some cluster prefixed control messages during startup diff --git a/network/p2p/inspector/validation/control_message_validation_inspector.go b/network/p2p/inspector/validation/control_message_validation_inspector.go index 5556edf685d..fdbaadc5e0d 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector.go @@ -116,10 +116,10 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid return nil, fmt.Errorf("failed to create cluster prefix topics received tracker") } - if params.Config.MessageMaxSampleSize < params.Config.MessageErrorThreshold { + if params.Config.PublishMessages.MaxSampleSize < params.Config.PublishMessages.ErrorThreshold { return nil, fmt.Errorf("rpc message max sample size must be greater than or equal to rpc message error threshold, got %d and %d respectively", - params.Config.MessageMaxSampleSize, - params.Config.MessageErrorThreshold) + params.Config.PublishMessages.MaxSampleSize, + params.Config.PublishMessages.ErrorThreshold) } c := &ControlMsgValidationInspector{ @@ -135,7 +135,7 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid topicOracle: params.TopicOracle, } - store := queue.NewHeroStore(params.Config.QueueSize, params.Logger, inspectMsgQueueCacheCollector) + store := queue.NewHeroStore(params.Config.InspectionQueue.Size, params.Logger, inspectMsgQueueCacheCollector) pool := worker.NewWorkerPoolBuilder[*InspectRPCRequest](lg, store, c.processInspectRPCReq).Build() @@ -158,7 +158,7 @@ func NewControlMsgValidationInspector(params *InspectorParams) (*ControlMsgValid <-c.distributor.Done() c.logger.Debug().Msg("rpc inspector distributor shutdown complete") }) - for i := 0; i < c.config.NumberOfWorkers; i++ { + for i := 0; i < c.config.InspectionQueue.NumberOfWorkers; i++ { builder.AddWorker(pool.WorkerLogic()) } c.Component = builder.Build() @@ -326,15 +326,27 @@ func (c *ControlMsgValidationInspector) checkPubsubMessageSender(message *pubsub // - error: if any error occurs while sampling or validating topics, all returned errors are benign and should not cause the node to crash. // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, grafts []*pubsub_pb.ControlGraft, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { - tracker := make(duplicateStrTracker) + duplicateTopicTracker := make(duplicateStrTracker) + totalDuplicateTopicIds := 0 + defer func() { + // regardless of inspection result, update metrics + c.metrics.OnGraftMessageInspected(totalDuplicateTopicIds) + }() for _, graft := range grafts { topic := channels.Topic(graft.GetTopicID()) - if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType + if duplicateTopicTracker.track(topic.String()) > 1 { + // ideally, a GRAFT message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. + totalDuplicateTopicIds++ + // check if the total number of duplicates exceeds the configured threshold. + if totalDuplicateTopicIds > c.config.GraftPrune.DuplicateTopicIdThreshold { + c.metrics.OnGraftDuplicateTopicIdsExceedThreshold() + return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgGraft), p2p.CtrlMsgNonClusterTopicType + } } - tracker.set(topic.String()) err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { + // TODO: consider adding a threshold for this error similar to the duplicate topic id threshold. + c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.CtrlMsgGraft) return err, ctrlMsgType } } @@ -353,14 +365,26 @@ func (c *ControlMsgValidationInspector) inspectGraftMessages(from peer.ID, graft // - bool: true if an error is returned and the topic that failed validation was a cluster prefixed topic, false otherwise. func (c *ControlMsgValidationInspector) inspectPruneMessages(from peer.ID, prunes []*pubsub_pb.ControlPrune, activeClusterIDS flow.ChainIDList) (error, p2p.CtrlMsgTopicType) { tracker := make(duplicateStrTracker) + totalDuplicateTopicIds := 0 + defer func() { + // regardless of inspection result, update metrics + c.metrics.OnPruneMessageInspected(totalDuplicateTopicIds) + }() for _, prune := range prunes { topic := channels.Topic(prune.GetTopicID()) - if tracker.isDuplicate(topic.String()) { - return NewDuplicateTopicErr(topic.String(), p2pmsg.CtrlMsgPrune), p2p.CtrlMsgNonClusterTopicType + if tracker.track(topic.String()) > 1 { + // ideally, a PRUNE message should not have any duplicate topics, hence a topic ID is counted as a duplicate only if it is repeated more than once. + totalDuplicateTopicIds++ + // check if the total number of duplicates exceeds the configured threshold. + if totalDuplicateTopicIds > c.config.GraftPrune.DuplicateTopicIdThreshold { + c.metrics.OnPruneDuplicateTopicIdsExceedThreshold() + return NewDuplicateTopicErr(topic.String(), totalDuplicateTopicIds, p2pmsg.CtrlMsgPrune), p2p.CtrlMsgNonClusterTopicType + } } - tracker.set(topic.String()) err, ctrlMsgType := c.validateTopic(from, topic, activeClusterIDS) if err != nil { + // TODO: consider adding a threshold for this error similar to the duplicate topic id threshold. + c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.CtrlMsgPrune) return err, ctrlMsgType } } @@ -384,32 +408,55 @@ func (c *ControlMsgValidationInspector) inspectIHaveMessages(from peer.ID, ihave lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). Int("sample_size", len(ihaves)). - Int("max_sample_size", c.config.IHave.MaxSampleSize). + Int("max_sample_size", c.config.IHave.MessageCountThreshold). Logger() duplicateTopicTracker := make(duplicateStrTracker) duplicateMessageIDTracker := make(duplicateStrTracker) totalMessageIds := 0 + totalDuplicateTopicIds := 0 + totalDuplicateMessageIds := 0 + defer func() { + // regardless of inspection result, update metrics + c.metrics.OnIHaveMessagesInspected(totalDuplicateTopicIds, totalDuplicateMessageIds) + }() for _, ihave := range ihaves { messageIds := ihave.GetMessageIDs() topic := ihave.GetTopicID() - if duplicateTopicTracker.isDuplicate(topic) { - return NewDuplicateTopicErr(topic, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType - } - duplicateTopicTracker.set(topic) + totalMessageIds += len(messageIds) + + // first check if the topic is valid, fail fast if it is not err, ctrlMsgType := c.validateTopic(from, channels.Topic(topic), activeClusterIDS) if err != nil { + // TODO: consider adding a threshold for this error similar to the duplicate topic id threshold. + c.metrics.OnInvalidTopicIdDetectedForControlMessage(p2pmsg.CtrlMsgIHave) return err, ctrlMsgType } + // then track the topic ensuring it is not beyond a duplicate threshold. + if duplicateTopicTracker.track(topic) > 1 { + totalDuplicateTopicIds++ + // the topic is duplicated, check if the total number of duplicates exceeds the configured threshold + if totalDuplicateTopicIds > c.config.IHave.DuplicateTopicIdThreshold { + c.metrics.OnIHaveDuplicateTopicIdsExceedThreshold() + return NewDuplicateTopicErr(topic, totalDuplicateTopicIds, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType + } + } + for _, messageID := range messageIds { - if duplicateMessageIDTracker.isDuplicate(messageID) { - return NewDuplicateTopicErr(messageID, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType + if duplicateMessageIDTracker.track(messageID) > 1 { + totalDuplicateMessageIds++ + // the message is duplicated, check if the total number of duplicates exceeds the configured threshold + if totalDuplicateMessageIds > c.config.IHave.DuplicateMessageIdThreshold { + c.metrics.OnIHaveDuplicateMessageIdsExceedThreshold() + return NewDuplicateMessageIDErr(messageID, totalDuplicateMessageIds, p2pmsg.CtrlMsgIHave), p2p.CtrlMsgNonClusterTopicType + } } - duplicateMessageIDTracker.set(messageID) } } lg.Debug(). Int("total_message_ids", totalMessageIds). + Int("total_duplicate_topic_ids", totalDuplicateTopicIds). + Int("total_duplicate_message_ids", totalDuplicateMessageIds). Msg("ihave control message validation complete") return nil, p2p.CtrlMsgNonClusterTopicType } @@ -432,20 +479,21 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant lastHighest := c.rpcTracker.LastHighestIHaveRPCSize() lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). - Uint("max_sample_size", c.config.IWant.MaxSampleSize). + Uint("max_sample_size", c.config.IWant.MessageCountThreshold). Int64("last_highest_ihave_rpc_size", lastHighest). Logger() - sampleSize := uint(len(iWants)) - tracker := make(duplicateStrTracker) + duplicateMsgIdTracker := make(duplicateStrTracker) cacheMisses := 0 - allowedCacheMissesThreshold := float64(sampleSize) * c.config.IWant.CacheMissThreshold - duplicates := 0 - allowedDuplicatesThreshold := float64(sampleSize) * c.config.IWant.DuplicateMsgIDThreshold - checkCacheMisses := len(iWants) >= c.config.IWant.CacheMissCheckSize + duplicateMessageIds := 0 + defer func() { + // regardless of inspection result, update metrics + c.metrics.OnIWantMessagesInspected(duplicateMessageIds, cacheMisses) + }() + lg = lg.With(). - Uint("iwant_sample_size", sampleSize). - Float64("allowed_cache_misses_threshold", allowedCacheMissesThreshold). - Float64("allowed_duplicates_threshold", allowedDuplicatesThreshold).Logger() + Int("iwant_msg_count", len(iWants)). + Int("cache_misses_threshold", c.config.IWant.CacheMissThreshold). + Int("duplicates_threshold", c.config.IWant.DuplicateMsgIdThreshold).Logger() lg.Trace().Msg("validating sample of message ids from iwant control message") @@ -455,22 +503,23 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant messageIDCount := uint(len(messageIds)) for _, messageID := range messageIds { // check duplicate allowed threshold - if tracker.isDuplicate(messageID) { - duplicates++ - if float64(duplicates) > allowedDuplicatesThreshold { - return NewIWantDuplicateMsgIDThresholdErr(duplicates, messageIDCount, c.config.IWant.DuplicateMsgIDThreshold) + if duplicateMsgIdTracker.track(messageID) > 1 { + // ideally, an iWant message should not have any duplicate message IDs, hence a message id is considered duplicate when it is repeated more than once. + duplicateMessageIds++ + if duplicateMessageIds > c.config.IWant.DuplicateMsgIdThreshold { + c.metrics.OnIWantDuplicateMessageIdsExceedThreshold() + return NewIWantDuplicateMsgIDThresholdErr(duplicateMessageIds, messageIDCount, c.config.IWant.DuplicateMsgIdThreshold) } } // check cache miss threshold if !c.rpcTracker.WasIHaveRPCSent(messageID) { cacheMisses++ - if checkCacheMisses { - if float64(cacheMisses) > allowedCacheMissesThreshold { - return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWant.CacheMissThreshold) - } + if cacheMisses > c.config.IWant.CacheMissThreshold { + c.metrics.OnIWantCacheMissMessageIdsExceedThreshold() + return NewIWantCacheMissThresholdErr(cacheMisses, messageIDCount, c.config.IWant.CacheMissThreshold) } } - tracker.set(messageID) + duplicateMsgIdTracker.track(messageID) totalMessageIds++ } } @@ -478,7 +527,7 @@ func (c *ControlMsgValidationInspector) inspectIWantMessages(from peer.ID, iWant lg.Debug(). Int("total_message_ids", totalMessageIds). Int("cache_misses", cacheMisses). - Int("duplicates", duplicates). + Int("total_duplicate_message_ids", duplicateMessageIds). Msg("iwant control message validation complete") return nil @@ -501,7 +550,7 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, if totalMessages == 0 { return nil, 0 } - sampleSize := c.config.MessageMaxSampleSize + sampleSize := c.config.PublishMessages.MaxSampleSize if sampleSize > totalMessages { sampleSize = totalMessages } @@ -519,10 +568,22 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, return false } var errs *multierror.Error + invalidTopicIdsCount := 0 + invalidSubscriptionsCount := 0 + invalidSendersCount := 0 + defer func() { + // regardless of inspection result, update metrics + errCnt := 0 + if errs != nil { + errCnt = errs.Len() + } + c.metrics.OnPublishMessageInspected(errCnt, invalidTopicIdsCount, invalidSubscriptionsCount, invalidSendersCount) + }() for _, message := range messages[:sampleSize] { if c.networkingType == network.PrivateNetwork { err := c.checkPubsubMessageSender(message) if err != nil { + invalidSendersCount++ errs = multierror.Append(errs, err) continue } @@ -534,17 +595,20 @@ func (c *ControlMsgValidationInspector) inspectRpcPublishMessages(from peer.ID, err, _ := c.validateTopic(from, topic, activeClusterIDS) if err != nil { // we can skip checking for subscription of topic that failed validation and continue + invalidTopicIdsCount++ errs = multierror.Append(errs, err) continue } if !hasSubscription(topic.String()) { + invalidSubscriptionsCount++ errs = multierror.Append(errs, fmt.Errorf("subscription for topic %s not found", topic)) } } // return an error when we exceed the error threshold - if errs != nil && errs.Len() > c.config.MessageErrorThreshold { + if errs != nil && errs.Len() > c.config.PublishMessages.ErrorThreshold { + c.metrics.OnPublishMessagesInspectionErrorExceedsThreshold() return NewInvalidRpcPublishMessagesErr(errs.ErrorOrNil(), errs.Len()), uint64(errs.Len()) } @@ -580,12 +644,12 @@ func (c *ControlMsgValidationInspector) truncateRPC(from peer.ID, rpc *pubsub.RP func (c *ControlMsgValidationInspector) truncateGraftMessages(rpc *pubsub.RPC) { grafts := rpc.GetControl().GetGraft() originalGraftSize := len(grafts) - if originalGraftSize <= c.config.GraftPruneMessageMaxSampleSize { + if originalGraftSize <= c.config.GraftPrune.MessageCountThreshold { return // nothing to truncate } // truncate grafts and update metrics - sampleSize := c.config.GraftPruneMessageMaxSampleSize + sampleSize := c.config.GraftPrune.MessageCountThreshold c.performSample(p2pmsg.CtrlMsgGraft, uint(originalGraftSize), uint(sampleSize), func(i, j uint) { grafts[i], grafts[j] = grafts[j], grafts[i] }) @@ -600,11 +664,11 @@ func (c *ControlMsgValidationInspector) truncateGraftMessages(rpc *pubsub.RPC) { func (c *ControlMsgValidationInspector) truncatePruneMessages(rpc *pubsub.RPC) { prunes := rpc.GetControl().GetPrune() originalPruneSize := len(prunes) - if originalPruneSize <= c.config.GraftPruneMessageMaxSampleSize { + if originalPruneSize <= c.config.GraftPrune.MessageCountThreshold { return // nothing to truncate } - sampleSize := c.config.GraftPruneMessageMaxSampleSize + sampleSize := c.config.GraftPrune.MessageCountThreshold c.performSample(p2pmsg.CtrlMsgPrune, uint(originalPruneSize), uint(sampleSize), func(i, j uint) { prunes[i], prunes[j] = prunes[j], prunes[i] }) @@ -613,7 +677,7 @@ func (c *ControlMsgValidationInspector) truncatePruneMessages(rpc *pubsub.RPC) { } // truncateIHaveMessages truncates the iHaves control messages in the RPC. If the total number of iHaves in the RPC exceeds the configured -// MaxSampleSize the list of iHaves will be truncated. +// MessageCountThreshold the list of iHaves will be truncated. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIHaveMessages(rpc *pubsub.RPC) { @@ -623,9 +687,9 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessages(rpc *pubsub.RPC) { return } - if originalIHaveCount > c.config.IHave.MaxSampleSize { + if originalIHaveCount > c.config.IHave.MessageCountThreshold { // truncate ihaves and update metrics - sampleSize := c.config.IHave.MaxSampleSize + sampleSize := c.config.IHave.MessageCountThreshold if sampleSize > originalIHaveCount { sampleSize = originalIHaveCount } @@ -639,7 +703,7 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessages(rpc *pubsub.RPC) { } // truncateIHaveMessageIds truncates the message ids for each iHave control message in the RPC. If the total number of message ids in a single iHave exceeds the configured -// MaxMessageIDSampleSize the list of message ids will be truncated. Before message ids are truncated the iHave control messages should have been truncated themselves. +// MessageIdCountThreshold the list of message ids will be truncated. Before message ids are truncated the iHave control messages should have been truncated themselves. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIHaveMessageIds(rpc *pubsub.RPC) { @@ -650,8 +714,8 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessageIds(rpc *pubsub.RPC) continue // nothing to truncate; skip } - if originalMessageIdCount > c.config.IHave.MaxMessageIDSampleSize { - sampleSize := c.config.IHave.MaxMessageIDSampleSize + if originalMessageIdCount > c.config.IHave.MessageIdCountThreshold { + sampleSize := c.config.IHave.MessageIdCountThreshold if sampleSize > originalMessageIdCount { sampleSize = originalMessageIdCount } @@ -666,7 +730,7 @@ func (c *ControlMsgValidationInspector) truncateIHaveMessageIds(rpc *pubsub.RPC) } // truncateIWantMessages truncates the iWant control messages in the RPC. If the total number of iWants in the RPC exceeds the configured -// MaxSampleSize the list of iWants will be truncated. +// MessageCountThreshold the list of iWants will be truncated. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIWantMessages(from peer.ID, rpc *pubsub.RPC) { @@ -676,9 +740,9 @@ func (c *ControlMsgValidationInspector) truncateIWantMessages(from peer.ID, rpc return } - if originalIWantCount > c.config.IWant.MaxSampleSize { + if originalIWantCount > c.config.IWant.MessageCountThreshold { // truncate iWants and update metrics - sampleSize := c.config.IWant.MaxSampleSize + sampleSize := c.config.IWant.MessageCountThreshold if sampleSize > originalIWantCount { sampleSize = originalIWantCount } @@ -692,22 +756,22 @@ func (c *ControlMsgValidationInspector) truncateIWantMessages(from peer.ID, rpc } // truncateIWantMessageIds truncates the message ids for each iWant control message in the RPC. If the total number of message ids in a single iWant exceeds the configured -// MaxMessageIDSampleSize the list of message ids will be truncated. Before message ids are truncated the iWant control messages should have been truncated themselves. +// MessageIdCountThreshold the list of message ids will be truncated. Before message ids are truncated the iWant control messages should have been truncated themselves. // Args: // - rpc: the rpc message to truncate. func (c *ControlMsgValidationInspector) truncateIWantMessageIds(from peer.ID, rpc *pubsub.RPC) { lastHighest := c.rpcTracker.LastHighestIHaveRPCSize() lg := c.logger.With(). Str("peer_id", p2plogging.PeerId(from)). - Uint("max_sample_size", c.config.IWant.MaxSampleSize). + Uint("max_sample_size", c.config.IWant.MessageCountThreshold). Int64("last_highest_ihave_rpc_size", lastHighest). Logger() sampleSize := int(10 * lastHighest) - if sampleSize == 0 || sampleSize > c.config.IWant.MaxMessageIDSampleSize { + if sampleSize == 0 || sampleSize > c.config.IWant.MessageIdCountThreshold { // invalid or 0 sample size is suspicious lg.Warn().Str(logging.KeySuspicious, "true").Msg("zero or invalid sample size, using default max sample size") - sampleSize = c.config.IWant.MaxMessageIDSampleSize + sampleSize = c.config.IWant.MessageIdCountThreshold } for _, iWant := range rpc.GetControl().GetIwant() { messageIDs := iWant.GetMessageIDs() @@ -874,8 +938,10 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In switch { case IsErrActiveClusterIDsNotSet(err): + c.metrics.OnActiveClusterIDsNotSetErr() lg.Warn().Msg("active cluster ids not set") case IsErrUnstakedPeer(err): + c.metrics.OnUnstakedPeerInspectionFailed() lg.Warn().Msg("control message received from unstaked peer") default: distErr := c.distributor.Distribute(p2p.NewInvalidControlMessageNotification(req.Peer, ctlMsgType, err, count, topicType)) @@ -883,8 +949,10 @@ func (c *ControlMsgValidationInspector) logAndDistributeAsyncInspectErrs(req *In lg.Error(). Err(distErr). Msg("failed to distribute invalid control message notification") + return } lg.Error().Msg("rpc control message async inspection failed") + c.metrics.OnInvalidControlMessageNotificationSent() } } diff --git a/network/p2p/inspector/validation/control_message_validation_inspector_test.go b/network/p2p/inspector/validation/control_message_validation_inspector_test.go index f9268224dee..672f89c057e 100644 --- a/network/p2p/inspector/validation/control_message_validation_inspector_test.go +++ b/network/p2p/inspector/validation/control_message_validation_inspector_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "sync" "testing" "time" @@ -29,6 +30,7 @@ import ( ) func TestNewControlMsgValidationInspector(t *testing.T) { + t.Run("should create validation inspector without error", func(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() @@ -83,16 +85,18 @@ func TestNewControlMsgValidationInspector(t *testing.T) { // Message truncation for each control message type occurs when the count of control // messages exceeds the configured maximum sample size for that control message type. func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { - t.Run("truncateGraftMessages should truncate graft messages as expected", func(t *testing.T) { + t.Run("graft truncation", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.GraftPruneMessageMaxSampleSize = graftPruneMessageMaxSampleSize + params.Config.GraftPrune.MessageCountThreshold = graftPruneMessageMaxSampleSize }) // topic validation is ignored set any topic oracle distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + // topic validation not performed so we can use random strings graftsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixtures(unittest.IdentifierListFixture(2000).Strings()...)...)) require.Greater(t, len(graftsGreaterThanMaxSampleSize.GetControl().GetGraft()), graftPruneMessageMaxSampleSize) @@ -109,13 +113,14 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { shouldNotBeTruncated := len(graftsLessThanMaxSampleSize.GetControl().GetGraft()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - t.Run("truncatePruneMessages should truncate prune messages as expected", func(t *testing.T) { + t.Run("prune truncation", func(t *testing.T) { graftPruneMessageMaxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.GraftPruneMessageMaxSampleSize = graftPruneMessageMaxSampleSize + params.Config.GraftPrune.MessageCountThreshold = graftPruneMessageMaxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() @@ -123,6 +128,8 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + // unittest.RequireCloseBefore(t, inspector.Ready(), 100*time.Millisecond, "inspector did not start") // topic validation not performed, so we can use random strings prunesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithPrunes(unittest.P2PRPCPruneFixtures(unittest.IdentifierListFixture(2000).Strings()...)...)) @@ -139,19 +146,21 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { shouldNotBeTruncated := len(prunesLessThanMaxSampleSize.GetControl().GetPrune()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - t.Run("truncateIHaveMessages should truncate iHave messages as expected", func(t *testing.T) { + t.Run("ihave message id truncation", func(t *testing.T) { maxSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IHave.MaxSampleSize = maxSampleSize + params.Config.IHave.MessageCountThreshold = maxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) // topic validation not performed so we can use random strings iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(2000, unittest.IdentifierListFixture(2000).Strings()...)...)) @@ -163,25 +172,27 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iHavesGreaterThanMaxSampleSize)) require.NoError(t, inspector.Inspect(from, iHavesLessThanMaxSampleSize)) require.Eventually(t, func() bool { - // rpc with iHaves greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iHaves greater than configured max sample size should be truncated to MessageCountThreshold shouldBeTruncated := len(iHavesGreaterThanMaxSampleSize.GetControl().GetIhave()) == maxSampleSize - // rpc with iHaves less than MaxSampleSize should not be truncated + // rpc with iHaves less than MessageCountThreshold should not be truncated shouldNotBeTruncated := len(iHavesLessThanMaxSampleSize.GetControl().GetIhave()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - t.Run("truncateIHaveMessageIds should truncate iHave message ids as expected", func(t *testing.T) { + t.Run("ihave message ids truncation", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IHave.MaxMessageIDSampleSize = maxMessageIDSampleSize + params.Config.IHave.MessageIdCountThreshold = maxMessageIDSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Twice() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) // topic validation not performed so we can use random strings iHavesGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIHaves(unittest.P2PRPCIHaveFixtures(2000, unittest.IdentifierListFixture(10).Strings()...)...)) @@ -192,32 +203,35 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iHavesLessThanMaxSampleSize)) require.Eventually(t, func() bool { for _, iHave := range iHavesGreaterThanMaxSampleSize.GetControl().GetIhave() { - // rpc with iHaves message ids greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iHaves message ids greater than configured max sample size should be truncated to MessageCountThreshold if len(iHave.GetMessageIDs()) != maxMessageIDSampleSize { return false } } for _, iHave := range iHavesLessThanMaxSampleSize.GetControl().GetIhave() { - // rpc with iHaves message ids less than MaxSampleSize should not be truncated + // rpc with iHaves message ids less than MessageCountThreshold should not be truncated if len(iHave.GetMessageIDs()) != 50 { return false } } return true }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - t.Run("truncateIWantMessages should truncate iWant messages as expected", func(t *testing.T) { + t.Run("iwant message truncation", func(t *testing.T) { maxSampleSize := uint(100) inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWant.MaxSampleSize = maxSampleSize + params.Config.IWant.MessageCountThreshold = maxSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(200, 200)...)) require.Greater(t, uint(len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant())), maxSampleSize) iWantsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(50, 200)...)) @@ -227,25 +241,28 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iWantsGreaterThanMaxSampleSize)) require.NoError(t, inspector.Inspect(from, iWantsLessThanMaxSampleSize)) require.Eventually(t, func() bool { - // rpc with iWants greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iWants greater than configured max sample size should be truncated to MessageCountThreshold shouldBeTruncated := len(iWantsGreaterThanMaxSampleSize.GetControl().GetIwant()) == int(maxSampleSize) - // rpc with iWants less than MaxSampleSize should not be truncated + // rpc with iWants less than MessageCountThreshold should not be truncated shouldNotBeTruncated := len(iWantsLessThanMaxSampleSize.GetControl().GetIwant()) == 50 return shouldBeTruncated && shouldNotBeTruncated }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) - t.Run("truncateIWantMessageIds should truncate iWant message ids as expected", func(t *testing.T) { + t.Run("iwant message id truncation", func(t *testing.T) { maxMessageIDSampleSize := 1000 inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWant.MaxMessageIDSampleSize = maxMessageIDSampleSize + params.Config.IWant.MessageIdCountThreshold = maxMessageIDSampleSize }) // topic validation is ignored set any topic oracle rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Maybe() distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Maybe() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + iWantsGreaterThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 2000)...)) iWantsLessThanMaxSampleSize := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 50)...)) @@ -254,436 +271,732 @@ func TestControlMessageValidationInspector_truncateRPC(t *testing.T) { require.NoError(t, inspector.Inspect(from, iWantsLessThanMaxSampleSize)) require.Eventually(t, func() bool { for _, iWant := range iWantsGreaterThanMaxSampleSize.GetControl().GetIwant() { - // rpc with iWants message ids greater than configured max sample size should be truncated to MaxSampleSize + // rpc with iWants message ids greater than configured max sample size should be truncated to MessageCountThreshold if len(iWant.GetMessageIDs()) != maxMessageIDSampleSize { return false } } for _, iWant := range iWantsLessThanMaxSampleSize.GetControl().GetIwant() { - // rpc with iWants less than MaxSampleSize should not be truncated + // rpc with iWants less than MessageCountThreshold should not be truncated if len(iWant.GetMessageIDs()) != 50 { return false } } return true }, time.Second, 500*time.Millisecond) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) } -// TestControlMessageValidationInspector_processInspectRPCReq verifies the correct behavior of control message validation. -// It ensures that valid RPC control messages do not trigger erroneous invalid control message notifications, -// while all types of invalid control messages trigger expected notifications. -func TestControlMessageValidationInspector_processInspectRPCReq(t *testing.T) { - t.Run("processInspectRPCReq should not disseminate any invalid notification errors for valid RPC's", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, _, topicProviderOracle := inspectorFixture(t) - defer distributor.AssertNotCalled(t, "Distribute") +// TestControlMessageInspection_ValidRpc ensures inspector does not disseminate invalid control message notifications for a valid RPC. +func TestControlMessageInspection_ValidRpc(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, _, topicProviderOracle := inspectorFixture(t) + defer distributor.AssertNotCalled(t, "Distribute") - topics := []string{ - fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID), - fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID), - fmt.Sprintf("%s/%s", channels.SyncCommittee, sporkID), - fmt.Sprintf("%s/%s", channels.RequestChunks, sporkID), + topics := []string{ + fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID), + fmt.Sprintf("%s/%s", channels.PushBlocks, sporkID), + fmt.Sprintf("%s/%s", channels.SyncCommittee, sporkID), + fmt.Sprintf("%s/%s", channels.RequestChunks, sporkID), + } + // avoid unknown topics errors + topicProviderOracle.UpdateTopics(topics) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + grafts := unittest.P2PRPCGraftFixtures(topics...) + prunes := unittest.P2PRPCPruneFixtures(topics...) + ihaves := unittest.P2PRPCIHaveFixtures(50, topics...) + iwants := unittest.P2PRPCIWantFixtures(2, 50) + pubsubMsgs := unittest.GossipSubMessageFixtures(10, topics[0]) + + rpc := unittest.P2PRPCFixture( + unittest.WithGrafts(grafts...), + unittest.WithPrunes(prunes...), + unittest.WithIHaves(ihaves...), + unittest.WithIWants(iwants...), + unittest.WithPubsubMessages(pubsubMsgs...)) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + id, ok := args[0].(string) + require.True(t, ok) + for _, iwant := range iwants { + for _, messageID := range iwant.GetMessageIDs() { + if id == messageID { + return + } + } } - // avoid unknown topics errors - topicProviderOracle.UpdateTopics(topics) - inspector.Start(signalerCtx) - grafts := unittest.P2PRPCGraftFixtures(topics...) - prunes := unittest.P2PRPCPruneFixtures(topics...) - ihaves := unittest.P2PRPCIHaveFixtures(50, topics...) - iwants := unittest.P2PRPCIWantFixtures(2, 5) - pubsubMsgs := unittest.GossipSubMessageFixtures(10, topics[0]) - - // avoid cache misses for iwant messages. - iwants[0].MessageIDs = ihaves[0].MessageIDs[:10] - iwants[1].MessageIDs = ihaves[1].MessageIDs[11:20] - expectedMsgIds := make([]string, 0) - expectedMsgIds = append(expectedMsgIds, ihaves[0].MessageIDs...) - expectedMsgIds = append(expectedMsgIds, ihaves[1].MessageIDs...) - rpc := unittest.P2PRPCFixture( - unittest.WithGrafts(grafts...), - unittest.WithPrunes(prunes...), - unittest.WithIHaves(ihaves...), - unittest.WithIWants(iwants...), - unittest.WithPubsubMessages(pubsubMsgs...)) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - require.Contains(t, expectedMsgIds, id) - }) + require.Fail(t, "message id not found in iwant messages") + }) - from := unittest.PeerIdFixture(t) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + from := unittest.PeerIdFixture(t) + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestGraftInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for +// graft messages when the topic is invalid. +func TestGraftInspection_InvalidTopic(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + // create unknown topic + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) + unknownTopicGraft := unittest.P2PRPCGraftFixture(&unknownTopic) + malformedTopicGraft := unittest.P2PRPCGraftFixture(&malformedTopic) + invalidSporkIDTopicGraft := unittest.P2PRPCGraftFixture(&invalidSporkIDTopic) + + unknownTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(unknownTopicGraft)) + malformedTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(malformedTopicGraft)) + invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, unknownTopicReq)) + require.NoError(t, inspector.Inspect(from, malformedTopicReq)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestGraftInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate invalid control message notifications +// for a valid RPC with duplicate graft topic ids below the threshold. +func TestGraftInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var grafts []*pubsub_pb.ControlGraft + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { + grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +func TestGraftInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var grafts []*pubsub_pb.ControlGraft + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { + grafts = append(grafts, unittest.P2PRPCGraftFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, from, notification.PeerID) + require.Equal(t, p2pmsg.CtrlMsgGraft, notification.MsgType) + require.True(t, validation.IsDuplicateTopicErr(notification.Error)) }) - t.Run("processInspectRPCReq should disseminate invalid control message notification for control messages with duplicate topics", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{duplicateTopic}) - // create control messages with duplicate topic - grafts := []*pubsub_pb.ControlGraft{unittest.P2PRPCGraftFixture(&duplicateTopic), unittest.P2PRPCGraftFixture(&duplicateTopic)} - prunes := []*pubsub_pb.ControlPrune{unittest.P2PRPCPruneFixture(&duplicateTopic), unittest.P2PRPCPruneFixture(&duplicateTopic)} - ihaves := []*pubsub_pb.ControlIHave{unittest.P2PRPCIHaveFixture(&duplicateTopic, unittest.IdentifierListFixture(20).Strings()...), - unittest.P2PRPCIHaveFixture(&duplicateTopic, unittest.IdentifierListFixture(20).Strings()...)} - from := unittest.PeerIdFixture(t) - duplicateTopicGraftsRpc := unittest.P2PRPCFixture(unittest.WithGrafts(grafts...)) - duplicateTopicPrunesRpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) - duplicateTopicIHavesRpc := unittest.P2PRPCFixture(unittest.WithIHaves(ihaves...)) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(func(args mock.Arguments) { - notification, ok := args[0].(*p2p.InvCtrlMsgNotif) - require.True(t, ok) - require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") - require.Equal(t, from, notification.PeerID) - require.Contains(t, []p2pmsg.ControlMessageType{p2pmsg.CtrlMsgGraft, p2pmsg.CtrlMsgPrune, p2pmsg.CtrlMsgIHave}, notification.MsgType) - require.True(t, validation.IsDuplicateTopicErr(notification.Error)) - }) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) - inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - require.NoError(t, inspector.Inspect(from, duplicateTopicGraftsRpc)) - require.NoError(t, inspector.Inspect(from, duplicateTopicPrunesRpc)) - require.NoError(t, inspector.Inspect(from, duplicateTopicIHavesRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) +// TestPruneInspection_DuplicateTopicIds_AboveThreshold ensures inspector disseminates an invalid control message notification for +// prune messages when the number of duplicate topic ids is above the threshold. +func TestPruneInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var prunes []*pubsub_pb.ControlPrune + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // we need threshold + 1 to trigger the invalid control message notification; as the first duplicate topic id is not counted + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold+2; i++ { + prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(func(args mock.Arguments) { + notification, ok := args[0].(*p2p.InvCtrlMsgNotif) + require.True(t, ok) + require.Equal(t, notification.TopicType, p2p.CtrlMsgNonClusterTopicType, "expected p2p.CtrlMsgNonClusterTopicType notification type, no RPC with cluster prefixed topic sent in this test") + require.Equal(t, from, notification.PeerID) + require.Equal(t, p2pmsg.CtrlMsgPrune, notification.MsgType) + require.True(t, validation.IsDuplicateTopicErr(notification.Error)) }) - t.Run("inspectGraftMessages should disseminate invalid control message notification for invalid graft messages as expected", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) - unknownTopicGraft := unittest.P2PRPCGraftFixture(&unknownTopic) - malformedTopicGraft := unittest.P2PRPCGraftFixture(&malformedTopic) - invalidSporkIDTopicGraft := unittest.P2PRPCGraftFixture(&invalidSporkIDTopic) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) - unknownTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(unknownTopicGraft)) - malformedTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(malformedTopicGraft)) - invalidSporkIDTopicReq := unittest.P2PRPCFixture(unittest.WithGrafts(invalidSporkIDTopicGraft)) + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgGraft, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) +// TestPruneInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate invalid control message notifications +// for a valid RPC with duplicate prune topic ids below the threshold. +func TestPrueInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + duplicateTopic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{duplicateTopic}) + var prunes []*pubsub_pb.ControlPrune + cfg, err := config.DefaultConfig() + require.NoError(t, err) + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.GraftPrune.DuplicateTopicIdThreshold; i++ { + prunes = append(prunes, unittest.P2PRPCPruneFixture(&duplicateTopic)) + } + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPrunes(prunes...)) + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) - inspector.Start(signalerCtx) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 100*time.Millisecond, inspector) - require.NoError(t, inspector.Inspect(from, unknownTopicReq)) - require.NoError(t, inspector.Inspect(from, malformedTopicReq)) - require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicReq)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - t.Run("inspectPruneMessages should disseminate invalid control message notification for invalid prune messages as expected", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) - unknownTopicPrune := unittest.P2PRPCPruneFixture(&unknownTopic) - malformedTopicPrune := unittest.P2PRPCPruneFixture(&malformedTopic) - invalidSporkIDTopicPrune := unittest.P2PRPCPruneFixture(&invalidSporkIDTopic) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) - unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(unknownTopicPrune)) - malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(malformedTopicPrune)) - invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(invalidSporkIDTopicPrune)) +// TestPruneInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for +// prune messages when the topic is invalid. +func TestPruneInspection_InvalidTopic(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + // create unknown topic + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) + unknownTopicPrune := unittest.P2PRPCPruneFixture(&unknownTopic) + malformedTopicPrune := unittest.P2PRPCPruneFixture(&malformedTopic) + invalidSporkIDTopicPrune := unittest.P2PRPCPruneFixture(&invalidSporkIDTopic) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) + unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(unknownTopicPrune)) + malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(malformedTopicPrune)) + invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithPrunes(invalidSporkIDTopicPrune)) - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgPrune, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - inspector.Start(signalerCtx) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) - require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) - require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) + require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) + require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with invalid topics as expected", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - // create unknown topic - unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) - unknownTopicIhave := unittest.P2PRPCIHaveFixture(&unknownTopic, unittest.IdentifierListFixture(5).Strings()...) - malformedTopicIhave := unittest.P2PRPCIHaveFixture(&malformedTopic, unittest.IdentifierListFixture(5).Strings()...) - invalidSporkIDTopicIhave := unittest.P2PRPCIHaveFixture(&invalidSporkIDTopic, unittest.IdentifierListFixture(5).Strings()...) +// TestIHaveInspection_InvalidTopic ensures inspector disseminates an invalid control message notification for +// iHave messages when the topic is invalid. +func TestIHaveInspection_InvalidTopic(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + // create unknown topic + unknownTopic, malformedTopic, invalidSporkIDTopic := invalidTopics(t, sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{unknownTopic, malformedTopic, invalidSporkIDTopic}) + unknownTopicIhave := unittest.P2PRPCIHaveFixture(&unknownTopic, unittest.IdentifierListFixture(5).Strings()...) + malformedTopicIhave := unittest.P2PRPCIHaveFixture(&malformedTopic, unittest.IdentifierListFixture(5).Strings()...) + invalidSporkIDTopicIhave := unittest.P2PRPCIHaveFixture(&invalidSporkIDTopic, unittest.IdentifierListFixture(5).Strings()...) - unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(unknownTopicIhave)) - malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(malformedTopicIhave)) - invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(invalidSporkIDTopicIhave)) + unknownTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(unknownTopicIhave)) + malformedTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(malformedTopicIhave)) + invalidSporkIDTopicRpc := unittest.P2PRPCFixture(unittest.WithIHaves(invalidSporkIDTopicIhave)) - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) - inspector.Start(signalerCtx) + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, channels.IsInvalidTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Times(3).Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) - require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) - require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) + require.NoError(t, inspector.Inspect(from, unknownTopicRpc)) + require.NoError(t, inspector.Inspect(from, malformedTopicRpc)) + require.NoError(t, inspector.Inspect(from, invalidSporkIDTopicRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - t.Run("inspectIHaveMessages should disseminate invalid control message notification for iHave messages with duplicate message ids as expected", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) - validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) - // avoid unknown topics errors - topicProviderOracle.UpdateTopics([]string{validTopic}) - duplicateMsgID := unittest.IdentifierFixture() - msgIds := flow.IdentifierList{duplicateMsgID, duplicateMsgID, duplicateMsgID} - duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) - duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) +// TestIHaveInspection_DuplicateTopicIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for +// iHave messages when duplicate topic ids are below allowed threshold. +func TestIHaveInspection_DuplicateTopicIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) - require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) + cfg, err := config.DefaultConfig() + require.NoError(t, err) + validTopicIHave := unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...) + ihaves := []*pubsub_pb.ControlIHave{validTopicIHave} + // duplicate the valid topic id on other iHave messages but with different message ids + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold-1; i++ { + ihaves = append(ihaves, unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...)) + } + // creates an RPC with duplicate topic ids but different message ids + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(ihaves...)) + from := unittest.PeerIdFixture(t) - t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when duplicate message ids exceeds the allowed threshold", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) - // oracle must be set even though iWant messages do not have topic IDs - duplicateMsgID := unittest.IdentifierFixture() - duplicates := flow.IdentifierList{duplicateMsgID, duplicateMsgID} - msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() - duplicateMsgIDIWant := unittest.P2PRPCIWantFixture(msgIds...) + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - require.Contains(t, msgIds, id) - }) +// TestIHaveInspection_DuplicateTopicIds_AboveThreshold ensures inspector disseminate an invalid control message notification for +// iHave messages when duplicate topic ids are above allowed threshold. +func TestIHaveInspection_DuplicateTopicIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) - inspector.Start(signalerCtx) + cfg, err := config.DefaultConfig() + require.NoError(t, err) + validTopicIHave := unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...) + ihaves := []*pubsub_pb.ControlIHave{validTopicIHave} + // duplicate the valid topic id on other iHave messages but with different message ids up to the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateTopicIdThreshold+2; i++ { + ihaves = append(ihaves, unittest.P2PRPCIHaveFixture(&validTopic, unittest.IdentifierListFixture(5).Strings()...)) + } + // creates an RPC with duplicate topic ids but different message ids + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(ihaves...)) + from := unittest.PeerIdFixture(t) - require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) + // one notification should be disseminated for invalid messages when the number of duplicates exceeds the threshold + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateTopicErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - t.Run("inspectIWantMessages should disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold", func(t *testing.T) { - cacheMissCheckSize := 1000 - inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.IWant.CacheMissCheckSize = cacheMissCheckSize - // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - params.Config.IWant.CacheMissThreshold = .9 - }) - // oracle must be set even though iWant messages do not have topic IDs - inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(cacheMissCheckSize+1, 100)...)) + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - found := false - for _, iwant := range inspectMsgRpc.GetControl().GetIwant() { - for _, messageID := range iwant.GetMessageIDs() { - if id == messageID { - found = true - } +// TestIHaveInspection_DuplicateMessageIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for +// iHave messages when duplicate message ids are below allowed threshold. +func TestIHaveInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) + duplicateMsgID := unittest.IdentifierFixture() + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + msgIds := flow.IdentifierList{} + // includes as many duplicates as allowed by the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold; i++ { + msgIds = append(msgIds, duplicateMsgID) + } + duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) + from := unittest.PeerIdFixture(t) + + // no notification should be disseminated for valid messages as long as the number of duplicates is below the threshold + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestIHaveInspection_DuplicateMessageIds_AboveThreshold ensures inspector disseminates an invalid control message notification for +// iHave messages when duplicate message ids are above allowed threshold. +func TestIHaveInspection_DuplicateMessageIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t) + validTopic := fmt.Sprintf("%s/%s", channels.PushBlocks.String(), sporkID) + // avoid unknown topics errors + topicProviderOracle.UpdateTopics([]string{validTopic}) + duplicateMsgID := unittest.IdentifierFixture() + + cfg, err := config.DefaultConfig() + require.NoError(t, err) + msgIds := flow.IdentifierList{} + // includes as many duplicates as beyond the threshold + for i := 0; i < cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IHave.DuplicateMessageIdThreshold+2; i++ { + msgIds = append(msgIds, duplicateMsgID) + } + duplicateMsgIDIHave := unittest.P2PRPCIHaveFixture(&validTopic, append(msgIds, unittest.IdentifierListFixture(5)...).Strings()...) + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIHaves(duplicateMsgIDIHave)) + from := unittest.PeerIdFixture(t) + + // one notification should be disseminated for invalid messages when the number of duplicates exceeds the threshold + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIHave, validation.IsDuplicateMessageIDErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // TODO: this sleeps should be replaced with a queue size checker. + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestIWantInspection_DuplicateMessageIds_BelowThreshold ensures inspector does not disseminate an invalid control message notification for +// iWant messages when duplicate message ids are below allowed threshold. +func TestIWantInspection_DuplicateMessageIds_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) + // oracle must be set even though iWant messages do not have topic IDs + duplicateMsgID := unittest.IdentifierFixture() + duplicates := flow.IdentifierList{} + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // includes as many duplicates as allowed by the threshold + for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIdThreshold)-2; i++ { + duplicates = append(duplicates, duplicateMsgID) + } + msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() + duplicateMsgIDIWant := unittest.P2PRPCIWantFixture(msgIds...) + + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) + + from := unittest.PeerIdFixture(t) + distributor.AssertNotCalled(t, "Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, msgIds, id) + }).Maybe() // if iwant message ids count are not bigger than cache miss check size, this method is not called, anyway in this test we do not care about this method. + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestIWantInspection_DuplicateMessageIds_AboveThreshold ensures inspector disseminates invalid control message notifications for iWant messages when duplicate message ids exceeds allowed threshold. +func TestIWantInspection_DuplicateMessageIds_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t) + // oracle must be set even though iWant messages do not have topic IDs + duplicateMsgID := unittest.IdentifierFixture() + duplicates := flow.IdentifierList{} + cfg, err := config.DefaultConfig() + require.NoError(t, err) + // includes as many duplicates as allowed by the threshold + for i := 0; i < int(cfg.NetworkConfig.GossipSub.RpcInspector.Validation.IWant.DuplicateMsgIdThreshold)+2; i++ { + duplicates = append(duplicates, duplicateMsgID) + } + msgIds := append(duplicates, unittest.IdentifierListFixture(5)...).Strings() + duplicateMsgIDIWant := unittest.P2PRPCIWantFixture(msgIds...) + + duplicateMsgIDRpc := unittest.P2PRPCFixture(unittest.WithIWants(duplicateMsgIDIWant)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantDuplicateMsgIDThresholdErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(true).Run(func(args mock.Arguments) { + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, msgIds, id) + }).Maybe() // if iwant message ids count are not bigger than cache miss check size, this method is not called, anyway in this test we do not care about this method. + + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, duplicateMsgIDRpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestIWantInspection_CacheMiss_AboveThreshold ensures inspector disseminates invalid control message notifications for iWant messages when cache misses exceeds allowed threshold. +func TestIWantInspection_CacheMiss_AboveThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // set high cache miss threshold to ensure we only disseminate notification when it is exceeded + params.Config.IWant.CacheMissThreshold = 900 + }) + // 10 iwant messages, each with 100 message ids; total of 1000 message ids, which when imitated as cache misses should trigger notification dissemination. + inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixtures(10, 100)...)) + + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.CtrlMsgIWant, validation.IsIWantCacheMissThresholdErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() + // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold + allIwantsChecked := sync.WaitGroup{} + allIwantsChecked.Add(901) // 901 message ids + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { + defer allIwantsChecked.Done() + + id, ok := args[0].(string) + require.True(t, ok) + found := false + for _, iwant := range inspectMsgRpc.GetControl().GetIwant() { + for _, messageID := range iwant.GetMessageIDs() { + if id == messageID { + found = true } } - require.True(t, found) - }) + } + require.True(t, found) + }) - inspector.Start(signalerCtx) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) + unittest.RequireReturnsBefore(t, allIwantsChecked.Wait, 1*time.Second, "all iwant messages should be checked for cache misses") + + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +func TestIWantInspection_CacheMiss_BelowThreshold(t *testing.T) { + inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // set high cache miss threshold to ensure that we do not disseminate notification in this test + params.Config.IWant.CacheMissThreshold = 99 }) + // oracle must be set even though iWant messages do not have topic IDs + defer distributor.AssertNotCalled(t, "Distribute") - t.Run("inspectIWantMessages should not disseminate invalid control message notification for iWant messages when cache misses exceeds allowed threshold if cache miss check size not exceeded", - func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, rpcTracker, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - // if size of iwants not greater than 10 cache misses will not be checked - params.Config.IWant.CacheMissCheckSize = 10 - // set high cache miss threshold to ensure we only disseminate notification when it is exceeded - params.Config.IWant.CacheMissThreshold = .9 - }) - // oracle must be set even though iWant messages do not have topic IDs - defer distributor.AssertNotCalled(t, "Distribute") - - msgIds := unittest.IdentifierListFixture(100).Strings() - inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) - rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - // return false each time to eventually force a notification to be disseminated when the cache miss count finally exceeds the 90% threshold - rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { - id, ok := args[0].(string) - require.True(t, ok) - require.Contains(t, msgIds, id) - }) - - from := unittest.PeerIdFixture(t) - inspector.Start(signalerCtx) + msgIds := unittest.IdentifierListFixture(98).Strings() // one less than cache miss threshold + inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithIWants(unittest.P2PRPCIWantFixture(msgIds...))) + rpcTracker.On("LastHighestIHaveRPCSize").Return(int64(100)).Maybe() - require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) + allIwantsChecked := sync.WaitGroup{} + allIwantsChecked.Add(len(msgIds)) + // returns false each time to imitate cache misses; however, since the number of cache misses is below the threshold, no notification should be disseminated. + rpcTracker.On("WasIHaveRPCSent", mock.AnythingOfType("string")).Return(false).Run(func(args mock.Arguments) { + defer allIwantsChecked.Done() + id, ok := args[0].(string) + require.True(t, ok) + require.Contains(t, msgIds, id) + }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when invalid pubsub messages count greater than configured RpcMessageErrorThreshold", func(t *testing.T) { - errThreshold := 500 - inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.MessageErrorThreshold = errThreshold - }) - // create unknown topic - unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() - // create malformed topic - malformedTopic := channels.Topic("!@#$%^&**((").String() - // a topics spork ID is considered invalid if it does not match the current spork ID - invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())).String() - // create 10 normal messages - pubsubMsgs := unittest.GossipSubMessageFixtures(50, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) - // add 550 invalid messages to force notification dissemination - invalidMessageFixtures := []*pubsub_pb.Message{ - {Topic: &unknownTopic}, - {Topic: &malformedTopic}, - {Topic: &invalidSporkIDTopic}, - } - for i := 0; i < errThreshold+1; i++ { - pubsubMsgs = append(pubsubMsgs, invalidMessageFixtures[rand.Intn(len(invalidMessageFixtures))]) - } - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - topics := make([]string, len(pubsubMsgs)) - for i, msg := range pubsubMsgs { - topics[i] = *msg.Topic - } - // set topic oracle to return list of topics to avoid hasSubscription errors and force topic validation - topicProviderOracle.UpdateTopics(topics) - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + from := unittest.PeerIdFixture(t) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) - inspector.Start(signalerCtx) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) + unittest.RequireReturnsBefore(t, allIwantsChecked.Wait, 1*time.Second, "all iwant messages should be checked for cache misses") - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when subscription missing for topic", func(t *testing.T) { - errThreshold := 500 - inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - params.Config.MessageErrorThreshold = errThreshold - }) - pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) - from := unittest.PeerIdFixture(t) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + // waits one more second to ensure no notification is disseminated + time.Sleep(1 * time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_ExceedingErrThreshold ensures inspector disseminates invalid control message notifications for RPCs that exceed the configured error threshold. +func TestPublishMessageInspection_ExceedingErrThreshold(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, sporkID, _, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.PublishMessages.ErrorThreshold = errThreshold }) + // create unknown topic + unknownTopic := channels.Topic(fmt.Sprintf("%s/%s", unittest.IdentifierFixture(), sporkID)).String() + // create malformed topic + malformedTopic := channels.Topic("!@#$%^&**((").String() + // a topics spork ID is considered invalid if it does not match the current spork ID + invalidSporkIDTopic := channels.Topic(fmt.Sprintf("%s/%s", channels.PushBlocks, unittest.IdentifierFixture())).String() + // create 10 normal messages + pubsubMsgs := unittest.GossipSubMessageFixtures(50, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) + // add 550 invalid messages to force notification dissemination + invalidMessageFixtures := []*pubsub_pb.Message{ + {Topic: &unknownTopic}, + {Topic: &malformedTopic}, + {Topic: &invalidSporkIDTopic}, + } + for i := 0; i < errThreshold+1; i++ { + pubsubMsgs = append(pubsubMsgs, invalidMessageFixtures[rand.Intn(len(invalidMessageFixtures))]) + } + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + topics := make([]string, len(pubsubMsgs)) + for i, msg := range pubsubMsgs { + topics[i] = *msg.Topic + } + // set topic oracle to return list of topics to avoid hasSubscription errors and force topic validation + topicProviderOracle.UpdateTopics(topics) + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when publish messages contain no topic", func(t *testing.T) { - errThreshold := 500 - inspector, signalerCtx, cancel, distributor, _, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { - // 5 invalid pubsub messages will force notification dissemination - params.Config.MessageErrorThreshold = errThreshold - }) - pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, "") - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - topics := make([]string, len(pubsubMsgs)) - for i, msg := range pubsubMsgs { - topics[i] = *msg.Topic - } - // set topic oracle to return list of topics excluding first topic sent - from := unittest.PeerIdFixture(t) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_MissingSubscription ensures inspector disseminates invalid control message notifications for RPCs that the peer is not subscribed to. +func TestPublishMessageInspection_MissingSubscription(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, sporkID, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + params.Config.PublishMessages.ErrorThreshold = errThreshold }) - t.Run("inspectRpcPublishMessages should not inspect pubsub message sender on public networks", func(t *testing.T) { - inspector, signalerCtx, cancel, _, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t) - from := unittest.PeerIdFixture(t) - defer idProvider.AssertNotCalled(t, "ByPeerID", from) - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - topicProviderOracle.UpdateTopics([]string{topic}) - pubsubMsgs := unittest.GossipSubMessageFixtures(10, topic, unittest.WithFrom(from)) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID)) + from := unittest.PeerIdFixture(t) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestPublishMessageInspection_MissingTopic ensures inspector disseminates invalid control message notifications for published messages with missing topics. +func TestPublishMessageInspection_MissingTopic(t *testing.T) { + errThreshold := 500 + inspector, signalerCtx, cancel, distributor, _, _, _, _ := inspectorFixture(t, func(params *validation.InspectorParams) { + // 5 invalid pubsub messages will force notification dissemination + params.Config.PublishMessages.ErrorThreshold = errThreshold }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from unstaked peer", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - // override the inspector and params, run the inspector in private mode - params.NetworkingType = network.PrivateNetwork - }) - from := unittest.PeerIdFixture(t) - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - topicProviderOracle.UpdateTopics([]string{topic}) - // default RpcMessageErrorThreshold is 500, 501 messages should trigger a notification - pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) - idProvider.On("ByPeerID", from).Return(nil, false).Times(501) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + pubsubMsgs := unittest.GossipSubMessageFixtures(errThreshold+1, "") + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + for _, msg := range pubsubMsgs { + msg.Topic = nil + } + from := unittest.PeerIdFixture(t) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestRpcInspectionDeactivatedOnPublicNetwork ensures inspector does not inspect RPCs on public networks. +func TestRpcInspectionDeactivatedOnPublicNetwork(t *testing.T) { + inspector, signalerCtx, cancel, _, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t) + from := unittest.PeerIdFixture(t) + defer idProvider.AssertNotCalled(t, "ByPeerID", from) + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + topicProviderOracle.UpdateTopics([]string{topic}) + pubsubMsgs := unittest.GossipSubMessageFixtures(10, topic, unittest.WithFrom(from)) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_Unstaked_From ensures inspector disseminates invalid control message notifications for published messages from unstaked peers. +func TestPublishMessageInspection_Unstaked_From(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { + // override the inspector and params, run the inspector in private mode + params.NetworkingType = network.PrivateNetwork }) - t.Run("inspectRpcPublishMessages should disseminate invalid control message notification when message is from ejected peer", func(t *testing.T) { - inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { - // override the inspector and params, run the inspector in private mode - params.NetworkingType = network.PrivateNetwork - }) - from := unittest.PeerIdFixture(t) - id := unittest.IdentityFixture() - id.Ejected = true - topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) - topicProviderOracle.UpdateTopics([]string{topic}) - pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) - idProvider.On("ByPeerID", from).Return(id, true).Times(501) - rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) - checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) - distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) - inspector.Start(signalerCtx) - require.NoError(t, inspector.Inspect(from, rpc)) - // sleep for 1 second to ensure rpc's is processed - time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + from := unittest.PeerIdFixture(t) + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + topicProviderOracle.UpdateTopics([]string{topic}) + // default RpcMessageErrorThreshold is 500, 501 messages should trigger a notification + pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) + idProvider.On("ByPeerID", from).Return(nil, false).Times(501) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") +} + +// TestControlMessageInspection_Ejected_From ensures inspector disseminates invalid control message notifications for published messages from ejected peers. +func TestPublishMessageInspection_Ejected_From(t *testing.T) { + inspector, signalerCtx, cancel, distributor, _, sporkID, idProvider, topicProviderOracle := inspectorFixture(t, func(params *validation.InspectorParams) { + // override the inspector and params, run the inspector in private mode + params.NetworkingType = network.PrivateNetwork }) + from := unittest.PeerIdFixture(t) + id := unittest.IdentityFixture() + id.Ejected = true + topic := fmt.Sprintf("%s/%s", channels.TestNetworkChannel, sporkID) + topicProviderOracle.UpdateTopics([]string{topic}) + pubsubMsgs := unittest.GossipSubMessageFixtures(501, topic, unittest.WithFrom(from)) + idProvider.On("ByPeerID", from).Return(id, true).Times(501) + rpc := unittest.P2PRPCFixture(unittest.WithPubsubMessages(pubsubMsgs...)) + checkNotification := checkNotificationFunc(t, from, p2pmsg.RpcPublishMessage, validation.IsInvalidRpcPublishMessagesErr, p2p.CtrlMsgNonClusterTopicType) + distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) + inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + + require.NoError(t, inspector.Inspect(from, rpc)) + // sleep for 1 second to ensure rpc's is processed + time.Sleep(time.Second) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } // TestNewControlMsgValidationInspector_validateClusterPrefixedTopic ensures cluster prefixed topics are validated as expected. @@ -699,10 +1012,13 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) inspector.ActiveClustersChanged(flow.ChainIDList{clusterID, flow.ChainID(unittest.IdentifierFixture().String()), flow.ChainID(unittest.IdentifierFixture().String())}) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("validateClusterPrefixedTopic should not return error if cluster prefixed hard threshold not exceeded for unknown cluster ids", func(t *testing.T) { @@ -718,11 +1034,13 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin id := unittest.IdentityFixture() idProvider.On("ByPeerID", from).Return(id, true).Once() inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("validateClusterPrefixedTopic should return an error when sender is unstaked", func(t *testing.T) { @@ -736,10 +1054,13 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin inspectMsgRpc := unittest.P2PRPCFixture(unittest.WithGrafts(unittest.P2PRPCGraftFixture(&clusterPrefixedTopic))) inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) t.Run("validateClusterPrefixedTopic should return error if cluster prefixed hard threshold exceeded for unknown cluster ids", func(t *testing.T) { @@ -758,12 +1079,15 @@ func TestNewControlMsgValidationInspector_validateClusterPrefixedTopic(t *testin inspector.ActiveClustersChanged(flow.ChainIDList{flow.ChainID(unittest.IdentifierFixture().String())}) distributor.On("Distribute", mock.AnythingOfType("*p2p.InvCtrlMsgNotif")).Return(nil).Once().Run(checkNotification) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + for i := 0; i < 11; i++ { require.NoError(t, inspector.Inspect(from, inspectMsgRpc)) } // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") }) } @@ -779,6 +1103,8 @@ func TestControlMessageValidationInspector_ActiveClustersChanged(t *testing.T) { } inspector.ActiveClustersChanged(activeClusterIds) inspector.Start(signalerCtx) + unittest.RequireComponentsReadyBefore(t, 1*time.Second, inspector) + from := unittest.PeerIdFixture(t) for _, id := range activeClusterIds { topic := channels.Topic(fmt.Sprintf("%s/%s", channels.SyncCluster(id), sporkID)).String() @@ -787,7 +1113,8 @@ func TestControlMessageValidationInspector_ActiveClustersChanged(t *testing.T) { } // sleep for 1 second to ensure rpc's is processed time.Sleep(time.Second) - stopInspector(t, cancel, inspector) + cancel() + unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") } // invalidTopics returns 3 invalid topics. @@ -859,8 +1186,3 @@ func inspectorFixture(t *testing.T, opts ...func(params *validation.InspectorPar signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) return validationInspector, signalerCtx, cancel, distributor, rpcTracker, sporkID, idProvider, topicProviderOracle } - -func stopInspector(t *testing.T, cancel context.CancelFunc, inspector *validation.ControlMsgValidationInspector) { - cancel() - unittest.RequireCloseBefore(t, inspector.Done(), 5*time.Second, "inspector did not stop") -} diff --git a/network/p2p/inspector/validation/errors.go b/network/p2p/inspector/validation/errors.go index b08af405a23..bb73f9cba9b 100644 --- a/network/p2p/inspector/validation/errors.go +++ b/network/p2p/inspector/validation/errors.go @@ -12,15 +12,15 @@ import ( type IWantDuplicateMsgIDThresholdErr struct { duplicates int sampleSize uint - threshold float64 + threshold int } func (e IWantDuplicateMsgIDThresholdErr) Error() string { - return fmt.Sprintf("%d/%d iWant duplicate message ids exceeds the allowed threshold: %f", e.duplicates, e.sampleSize, e.threshold) + return fmt.Sprintf("%d/%d iWant duplicate message ids exceeds the allowed threshold: %d", e.duplicates, e.sampleSize, e.threshold) } // NewIWantDuplicateMsgIDThresholdErr returns a new IWantDuplicateMsgIDThresholdErr. -func NewIWantDuplicateMsgIDThresholdErr(duplicates int, sampleSize uint, threshold float64) IWantDuplicateMsgIDThresholdErr { +func NewIWantDuplicateMsgIDThresholdErr(duplicates int, sampleSize uint, threshold int) IWantDuplicateMsgIDThresholdErr { return IWantDuplicateMsgIDThresholdErr{duplicates, sampleSize, threshold} } @@ -34,15 +34,15 @@ func IsIWantDuplicateMsgIDThresholdErr(err error) bool { type IWantCacheMissThresholdErr struct { cacheMissCount int // total iwant cache misses sampleSize uint - threshold float64 + threshold int } func (e IWantCacheMissThresholdErr) Error() string { - return fmt.Sprintf("%d/%d iWant cache misses exceeds the allowed threshold: %f", e.cacheMissCount, e.sampleSize, e.threshold) + return fmt.Sprintf("%d/%d iWant cache misses exceeds the allowed threshold: %d", e.cacheMissCount, e.sampleSize, e.threshold) } // NewIWantCacheMissThresholdErr returns a new IWantCacheMissThresholdErr. -func NewIWantCacheMissThresholdErr(cacheMissCount int, sampleSize uint, threshold float64) IWantCacheMissThresholdErr { +func NewIWantCacheMissThresholdErr(cacheMissCount int, sampleSize uint, threshold int) IWantCacheMissThresholdErr { return IWantCacheMissThresholdErr{cacheMissCount, sampleSize, threshold} } @@ -54,8 +54,9 @@ func IsIWantCacheMissThresholdErr(err error) bool { // DuplicateTopicErr error that indicates a duplicate has been detected. This can be duplicate topic or message ID tracking. type DuplicateTopicErr struct { - topic string - msgType p2pmsg.ControlMessageType + topic string // the topic that is duplicated + count int // the number of times the topic has been duplicated + msgType p2pmsg.ControlMessageType // the control message type that the topic was found in } func (e DuplicateTopicErr) Error() string { @@ -63,8 +64,18 @@ func (e DuplicateTopicErr) Error() string { } // NewDuplicateTopicErr returns a new DuplicateTopicErr. -func NewDuplicateTopicErr(topic string, msgType p2pmsg.ControlMessageType) DuplicateTopicErr { - return DuplicateTopicErr{topic, msgType} +// Args: +// +// topic: the topic that is duplicated +// count: the number of times the topic has been duplicated +// msgType: the control message type that the topic was found in +// +// Returns: +// +// A new DuplicateTopicErr. +func NewDuplicateTopicErr(topic string, count int, msgType p2pmsg.ControlMessageType) DuplicateTopicErr { + + return DuplicateTopicErr{topic, count, msgType} } // IsDuplicateTopicErr returns true if an error is DuplicateTopicErr. @@ -75,8 +86,9 @@ func IsDuplicateTopicErr(err error) bool { // DuplicateMessageIDErr error that indicates a duplicate message ID has been detected in a IHAVE or IWANT control message. type DuplicateMessageIDErr struct { - id string - msgType p2pmsg.ControlMessageType + id string // id of the message that is duplicated + count int // the number of times the message ID has been duplicated + msgType p2pmsg.ControlMessageType // the control message type that the message ID was found in } func (e DuplicateMessageIDErr) Error() string { @@ -84,8 +96,13 @@ func (e DuplicateMessageIDErr) Error() string { } // NewDuplicateMessageIDErr returns a new DuplicateMessageIDErr. -func NewDuplicateMessageIDErr(id string, msgType p2pmsg.ControlMessageType) DuplicateMessageIDErr { - return DuplicateMessageIDErr{id, msgType} +// Args: +// +// id: id of the message that is duplicated +// count: the number of times the message ID has been duplicated +// msgType: the control message type that the message ID was found in. +func NewDuplicateMessageIDErr(id string, count int, msgType p2pmsg.ControlMessageType) DuplicateMessageIDErr { + return DuplicateMessageIDErr{id, count, msgType} } // IsDuplicateMessageIDErr returns true if an error is DuplicateMessageIDErr. diff --git a/network/p2p/inspector/validation/errors_test.go b/network/p2p/inspector/validation/errors_test.go index b042dd98bcf..29072fbd5f7 100644 --- a/network/p2p/inspector/validation/errors_test.go +++ b/network/p2p/inspector/validation/errors_test.go @@ -30,7 +30,7 @@ func TestErrActiveClusterIDsNotSetRoundTrip(t *testing.T) { // TestErrDuplicateTopicRoundTrip ensures correct error formatting for DuplicateTopicErr. func TestDuplicateTopicErrRoundTrip(t *testing.T) { expectedErrorMsg := fmt.Sprintf("duplicate topic found in %s control message type: %s", p2pmsg.CtrlMsgGraft, channels.TestNetworkChannel) - err := NewDuplicateTopicErr(channels.TestNetworkChannel.String(), p2pmsg.CtrlMsgGraft) + err := NewDuplicateTopicErr(channels.TestNetworkChannel.String(), 1, p2pmsg.CtrlMsgGraft) assert.Equal(t, expectedErrorMsg, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateTopicErr(err), "IsDuplicateTopicErr should return true for DuplicateTopicErr error") @@ -44,11 +44,11 @@ func TestDuplicateMessageIDErrRoundTrip(t *testing.T) { msgID := "flow-1804flkjnafo" expectedErrMsg1 := fmt.Sprintf("duplicate message ID foud in %s control message type: %s", p2pmsg.CtrlMsgIHave, msgID) expectedErrMsg2 := fmt.Sprintf("duplicate message ID foud in %s control message type: %s", p2pmsg.CtrlMsgIWant, msgID) - err := NewDuplicateMessageIDErr(msgID, p2pmsg.CtrlMsgIHave) + err := NewDuplicateMessageIDErr(msgID, 1, p2pmsg.CtrlMsgIHave) assert.Equal(t, expectedErrMsg1, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateMessageIDErr(err), "IsDuplicateMessageIDErr should return true for DuplicateMessageIDErr error") - err = NewDuplicateMessageIDErr(msgID, p2pmsg.CtrlMsgIWant) + err = NewDuplicateMessageIDErr(msgID, 1, p2pmsg.CtrlMsgIWant) assert.Equal(t, expectedErrMsg2, err.Error(), "the error message should be correctly formatted") // tests the IsDuplicateTopicErr function. assert.True(t, IsDuplicateMessageIDErr(err), "IsDuplicateMessageIDErr should return true for DuplicateMessageIDErr error") @@ -59,10 +59,10 @@ func TestDuplicateMessageIDErrRoundTrip(t *testing.T) { // TestIWantCacheMissThresholdErrRoundTrip ensures correct error formatting for IWantCacheMissThresholdErr. func TestIWantCacheMissThresholdErrRoundTrip(t *testing.T) { - err := NewIWantCacheMissThresholdErr(5, 10, .4) + err := NewIWantCacheMissThresholdErr(5, 10, 5) // tests the error message formatting. - expectedErrMsg := "5/10 iWant cache misses exceeds the allowed threshold: 0.400000" + expectedErrMsg := "5/10 iWant cache misses exceeds the allowed threshold: 5" assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") // tests the IsIWantCacheMissThresholdErr function. @@ -75,10 +75,10 @@ func TestIWantCacheMissThresholdErrRoundTrip(t *testing.T) { // TestIWantDuplicateMsgIDThresholdErrRoundTrip ensures correct error formatting for IWantDuplicateMsgIDThresholdErr. func TestIWantDuplicateMsgIDThresholdErrRoundTrip(t *testing.T) { - err := NewIWantDuplicateMsgIDThresholdErr(5, 10, .4) + err := NewIWantDuplicateMsgIDThresholdErr(5, 10, 5) // tests the error message formatting. - expectedErrMsg := "5/10 iWant duplicate message ids exceeds the allowed threshold: 0.400000" + expectedErrMsg := "5/10 iWant duplicate message ids exceeds the allowed threshold: 5" assert.Equal(t, expectedErrMsg, err.Error(), "the error message should be correctly formatted") // tests the IsIWantDuplicateMsgIDThresholdErr function. diff --git a/network/p2p/inspector/validation/utils.go b/network/p2p/inspector/validation/utils.go index 0e57a9dd345..d84bdd0cbf2 100644 --- a/network/p2p/inspector/validation/utils.go +++ b/network/p2p/inspector/validation/utils.go @@ -1,12 +1,24 @@ package validation -type duplicateStrTracker map[string]struct{} +// duplicateStrTracker is a map of strings to the number of times they have been tracked. +// It is a non-concurrent map, so it should only be used in a single goroutine. +// It is used to track duplicate strings. +type duplicateStrTracker map[string]int -func (d duplicateStrTracker) set(s string) { - d[s] = struct{}{} -} +// track stores the string and returns the number of times it has been tracked and whether it is a duplicate. +// If the string has not been tracked before, it is stored with a count of 1. +// If the string has been tracked before, the count is incremented. +// Args: +// +// s: the string to track +// +// Returns: +// The number of times this string has been tracked, e.g., 1 if it is the first time, 2 if it is the second time, etc. +func (d duplicateStrTracker) track(s string) int { + if _, ok := d[s]; !ok { + d[s] = 0 + } + d[s]++ -func (d duplicateStrTracker) isDuplicate(s string) bool { - _, ok := d[s] - return ok + return d[s] } diff --git a/network/p2p/inspector/validation/utils_test.go b/network/p2p/inspector/validation/utils_test.go index 14015dcc621..81f0ffca936 100644 --- a/network/p2p/inspector/validation/utils_test.go +++ b/network/p2p/inspector/validation/utils_test.go @@ -6,11 +6,17 @@ import ( "github.com/stretchr/testify/require" ) -// TestDuplicateStrTracker ensures duplicateStrTracker performs simple set and returns expected result for duplicate str. -func TestDuplicateStrTracker(t *testing.T) { +// TestDuplicateStringTracker tests the duplicateStrTracker.track function. +func TestDuplicateStringTracker(t *testing.T) { tracker := make(duplicateStrTracker) - s := "hello world" - require.False(t, tracker.isDuplicate(s)) - tracker.set(s) - require.True(t, tracker.isDuplicate(s)) + require.Equal(t, 1, tracker.track("test1")) + require.Equal(t, 2, tracker.track("test1")) + + // tracking a new string, 3 times + require.Equal(t, 1, tracker.track("test2")) + require.Equal(t, 2, tracker.track("test2")) + require.Equal(t, 3, tracker.track("test2")) + + // tracking an empty string + require.Equal(t, 1, tracker.track("")) } diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 50fb4cb80ca..10044db2631 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -933,6 +933,18 @@ func WithIHave(msgCount, msgIDCount int, topicId string) GossipSubCtrlOption { } } +// WithIHaveMessageIDs adds iHave control messages with the given message IDs to the control message. +func WithIHaveMessageIDs(msgIDs []string, topicId string) GossipSubCtrlOption { + return func(msg *pb.ControlMessage) { + msg.Ihave = []*pb.ControlIHave{ + { + TopicID: &topicId, + MessageIDs: msgIDs, + }, + } + } +} + // WithIWant adds iWant control messages of the given size and number to the control message. // The message IDs are generated randomly. // Args: