diff --git a/config.yml b/config.yml index 28133e55..39b2f624 100644 --- a/config.yml +++ b/config.yml @@ -72,15 +72,11 @@ multiplexer: transforms: normalize: qname-lowercase: true - reducer: - repetitive-traffic-detector: true - watch-interval: 5 loggers: - name: console stdout: mode: text - text-format: "timestamp-rfc3339ns identity operation rcode queryip qname qtype reducer-occurences reducer-cumulative-length" routes: - from: [ tap ] @@ -605,9 +601,12 @@ multiplexer: # # Use this transformer to detect trafic duplication # # additionnals directive for text format # # - reducer-occurences: number of occurences detected +# # - cumulative-length: sum of the length of each occurences # reducer: # # enable detector # repetitive-traffic-detector: true +# # limit to qname+1 instead of the complete qname to detect repetition +# qname-plus-one: false # # watch interval in seconds # watch-interval: 5 diff --git a/dnsutils/config.go b/dnsutils/config.go index cbc708b4..75cb6dfc 100644 --- a/dnsutils/config.go +++ b/dnsutils/config.go @@ -64,6 +64,7 @@ type ConfigTransformers struct { Reducer struct { Enable bool `yaml:"enable"` RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector"` + QnamePlusOne bool `yaml:"qname-plus-one"` WatchInterval int `yaml:"watch-interval"` } Filtering struct { @@ -130,6 +131,7 @@ func (c *ConfigTransformers) SetDefault() { c.Reducer.Enable = false c.Reducer.RepetitiveTrafficDetector = false + c.Reducer.QnamePlusOne = false c.Reducer.WatchInterval = 5 c.Filtering.Enable = false diff --git a/doc/transformers.md b/doc/transformers.md index 2ab72025..bbcfd1e0 100644 --- a/doc/transformers.md +++ b/doc/transformers.md @@ -277,10 +277,19 @@ Example of DNS messages in text format ### Traffic Reducer -Use this transformer to detect repetitive traffic +Use this transformer to detect repetitive traffic. +A query or reply is repeated when the following criterias are the same. + +The following criterias are used: +- server identity +- operation +- qname or qname+1 +- query ip +- qtype Options: - `repetitive-traffic-detector`: (boolean) detect repetitive traffic +- `qname-plus-one`: (boolean) use qname+1 instead of the complete one - `watch-interval`: (integer) watch interval in seconds Default values: @@ -289,11 +298,13 @@ Default values: transforms: reducer: repetitive-traffic-detector: true + qname-plus-one: false watch-interval: 5 ``` -Specific directive(s) available for the text format: +Specific text directive(s) available for the text format: - `reducer-occurences`: display the number of detected duplication +- `cumulative-length`: sum of the length of each occurences When the feature is enabled, the following json field are populated in your DNS message: @@ -303,6 +314,7 @@ Example: { "reducer": { "occurences": 1, + "cumulative-length": 47 } } ``` diff --git a/example-config/use-case-20.yml b/example-config/use-case-20.yml index 1b092049..91e1bb17 100644 --- a/example-config/use-case-20.yml +++ b/example-config/use-case-20.yml @@ -12,15 +12,17 @@ multiplexer: dnstap: listen-ip: 0.0.0.0 listen-port: 6000 + transforms: + reducer: + repetitive-traffic-detector: true + qname-plus-one: false + watch-interval: 5 loggers: - name: console stdout: mode: text - transforms: - reducer: - repetitive-traffic-detector: true - watch-interval: 5 + text-format: "timestamp-rfc3339ns identity operation rcode queryip qname qtype reducer-occurences reducer-cumulative-length" routes: - from: [ tap ] diff --git a/transformers/reducer.go b/transformers/reducer.go index 9232d8e1..2ef95ca0 100644 --- a/transformers/reducer.go +++ b/transformers/reducer.go @@ -8,6 +8,7 @@ import ( "github.com/dmachard/go-dnscollector/dnsutils" "github.com/dmachard/go-logger" + publicsuffixlist "golang.org/x/net/publicsuffix" ) type expiredKey struct { @@ -149,6 +150,13 @@ func (p *ReducerProcessor) RepetitiveTrafficDetector(dm *dnsutils.DnsMessage) in p.strBuilder.WriteString(dm.DnsTap.Identity) p.strBuilder.WriteString(dm.DnsTap.Operation) p.strBuilder.WriteString(dm.NetworkInfo.QueryIp) + if p.config.Reducer.QnamePlusOne { + qname := strings.ToLower(dm.DNS.Qname) + qname = strings.TrimSuffix(qname, ".") + if etld, err := publicsuffixlist.EffectiveTLDPlusOne(qname); err == nil { + dm.DNS.Qname = etld + } + } p.strBuilder.WriteString(dm.DNS.Qname) p.strBuilder.WriteString(dm.DNS.Qtype) dmTag := p.strBuilder.String() diff --git a/transformers/reducer_test.go b/transformers/reducer_test.go index b944bf7b..d5651d6b 100644 --- a/transformers/reducer_test.go +++ b/transformers/reducer_test.go @@ -30,7 +30,8 @@ func TestReducer_Json(t *testing.T) { refJson := ` { "reducer": { - "occurences": 0 + "occurences": 0, + "cumulative-length": 0 } } `