add qname+one criteria

config.yml

@@ -72,15 +72,11 @@ multiplexer:
       transforms:
         normalize:
           qname-lowercase: true
-        reducer:
-          repetitive-traffic-detector: true
-          watch-interval: 5
 
   loggers:
     - name: console
       stdout:
         mode: text
-        text-format: "timestamp-rfc3339ns identity operation rcode queryip qname qtype reducer-occurences reducer-cumulative-length"
 
   routes:
     - from: [ tap ]
@@ -605,9 +601,12 @@ multiplexer:
 # # Use this transformer to detect trafic duplication
 # # additionnals directive for text format
 # # - reducer-occurences: number of occurences detected
+# # - cumulative-length: sum of the length of each occurences
 # reducer:
 #   # enable detector
 #   repetitive-traffic-detector: true
+#   # limit to qname+1 instead of the complete qname to detect repetition
+#   qname-plus-one: false
 #   # watch interval in seconds
 #   watch-interval: 5
 

dnsutils/config.go

@@ -64,6 +64,7 @@ type ConfigTransformers struct {
 	Reducer struct {
 		Enable                    bool `yaml:"enable"`
 		RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector"`
+		QnamePlusOne              bool `yaml:"qname-plus-one"`
 		WatchInterval             int  `yaml:"watch-interval"`
 	}
 	Filtering struct {
@@ -130,6 +131,7 @@ func (c *ConfigTransformers) SetDefault() {
 
 	c.Reducer.Enable = false
 	c.Reducer.RepetitiveTrafficDetector = false
+	c.Reducer.QnamePlusOne = false
 	c.Reducer.WatchInterval = 5
 
 	c.Filtering.Enable = false

doc/transformers.md

@@ -277,10 +277,19 @@ Example of DNS messages in text format
 
 ### Traffic Reducer
 
-Use this transformer to detect repetitive traffic
+Use this transformer to detect repetitive traffic.
+A query or reply is repeated when the following criterias are the same. 
+
+The following criterias are used:
+- server identity
+- operation 
+- qname or qname+1
+- query ip
+- qtype
 
 Options:
 - `repetitive-traffic-detector`: (boolean) detect repetitive traffic
+- `qname-plus-one`: (boolean) use qname+1 instead of the complete one
 - `watch-interval`: (integer) watch interval in seconds
 
 Default values:
@@ -289,11 +298,13 @@ Default values:
 transforms:
   reducer:
     repetitive-traffic-detector: true
+    qname-plus-one: false
     watch-interval: 5
 ```
 
-Specific directive(s) available for the text format:
+Specific text directive(s) available for the text format:
 - `reducer-occurences`: display the number of detected duplication
+- `cumulative-length`: sum of the length of each occurences
 
 When the feature is enabled, the following json field are populated in your DNS message:
 
@@ -303,6 +314,7 @@ Example:
 {
   "reducer": {
     "occurences": 1,
+    "cumulative-length": 47
   }
 }
 ```

example-config/use-case-20.yml

@@ -12,15 +12,17 @@ multiplexer:
       dnstap:
         listen-ip: 0.0.0.0
         listen-port: 6000
+      transforms:
+        reducer:
+          repetitive-traffic-detector: true
+          qname-plus-one: false
+          watch-interval: 5
 
   loggers:
     - name: console
       stdout:
         mode: text
-      transforms:
-        reducer:
-          repetitive-traffic-detector: true
-          watch-interval: 5
+        text-format: "timestamp-rfc3339ns identity operation rcode queryip qname qtype reducer-occurences reducer-cumulative-length"
 
   routes:
     - from: [ tap ]

transformers/reducer.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/dmachard/go-dnscollector/dnsutils"
 	"github.com/dmachard/go-logger"
+	publicsuffixlist "golang.org/x/net/publicsuffix"
 )
 
 type expiredKey struct {
@@ -149,6 +150,13 @@ func (p *ReducerProcessor) RepetitiveTrafficDetector(dm *dnsutils.DnsMessage) in
 	p.strBuilder.WriteString(dm.DnsTap.Identity)
 	p.strBuilder.WriteString(dm.DnsTap.Operation)
 	p.strBuilder.WriteString(dm.NetworkInfo.QueryIp)
+	if p.config.Reducer.QnamePlusOne {
+		qname := strings.ToLower(dm.DNS.Qname)
+		qname = strings.TrimSuffix(qname, ".")
+		if etld, err := publicsuffixlist.EffectiveTLDPlusOne(qname); err == nil {
+			dm.DNS.Qname = etld
+		}
+	}
 	p.strBuilder.WriteString(dm.DNS.Qname)
 	p.strBuilder.WriteString(dm.DNS.Qtype)
 	dmTag := p.strBuilder.String()

transformers/reducer_test.go

@@ -30,7 +30,8 @@ func TestReducer_Json(t *testing.T) {
 	refJson := `
 			{
 				"reducer": {
-				  "occurences": 0
+				  "occurences": 0,
+				  "cumulative-length": 0
 				}
 			}
 			`