add custom text format with jinja templating (#748)

README.md

@@ -2,7 +2,7 @@
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.21-green" alt="Go version"/>
 <img src="https://img.shields.io/badge/go%20tests-439-green" alt="Go tests"/>
-<img src="https://img.shields.io/badge/go%20bench-20-green" alt="Go bench"/>
+<img src="https://img.shields.io/badge/go%20bench-21-green" alt="Go bench"/>
 <img src="https://img.shields.io/badge/go%20lines-30949-green" alt="Go lines"/>
 </p>
 

config.yml

@@ -6,10 +6,11 @@ global:
   trace:
     verbose: true
   server-identity: "dns-collector"
+  pid-file: ""
   text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport family protocol length-unit qname qtype latency"
   text-format-delimiter: " "
   text-format-boundary: "\""
-  pid-file: ""
+  text-jinja: ""
   worker:
     interval-monitor: 10
     buffer-size: 4096
@@ -46,7 +47,7 @@ pipelines:
 
   - name: console
     stdout:
-      mode: text
+      mode: jinja
 
 ################################################
 # DEPRECATED - multiplexer configuration

dnsutils/message.go

@@ -19,6 +19,7 @@ import (
 	"github.com/dmachard/go-dnscollector/pkgconfig"
 	"github.com/dmachard/go-dnstap-protobuf"
 	"github.com/dmachard/go-netutils"
+	"github.com/flosch/pongo2/v6"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/miekg/dns"
@@ -118,6 +119,8 @@ type DNS struct {
 	Qname   string `json:"qname"`
 	Qclass  string `json:"qclass"`
 
+	QuestionsCount int `json:"questions-count"`
+
 	Qtype           string   `json:"qtype"`
 	Flags           DNSFlags `json:"flags"`
 	DNSRRs          DNSRRs   `json:"resource-records"`
@@ -872,6 +875,22 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo
 	return []byte(s.String()), nil
 }
 
+func (dm *DNSMessage) ToTextTemplate(template string) (string, error) {
+	context := pongo2.Context{"dm": dm}
+
+	// Parse and execute the template
+	tmpl, err := pongo2.FromString(template)
+	if err != nil {
+		return "", err
+	}
+
+	result, err := tmpl.Execute(context)
+	if err != nil {
+		return "", err
+	}
+	return result, nil
+}
+
 func (dm *DNSMessage) ToJSON() string {
 	buffer := new(bytes.Buffer)
 	json.NewEncoder(buffer).Encode(dm)

dnsutils/message_test.go

@@ -213,6 +213,7 @@ func TestDnsMessage_Json_Reference(t *testing.T) {
 				  "qname": "-",
 				  "qtype": "-",
 				  "qclass": "-",
+				  "questions-count": 0,
 				  "flags": {
 					"qr": false,
 					"tc": false,
@@ -1556,3 +1557,62 @@ func BenchmarkDnsMessage_ToFlatten(b *testing.B) {
 		}
 	}
 }
+
+// To jinja templating
+func TestDnsMessage_ToJinjaFormat(t *testing.T) {
+	dm := DNSMessage{}
+	dm.Init()
+
+	dm.DNS.Qname = "qname_for_test"
+
+	template := `
+;; Got {% if dm.DNS.Type == "QUERY" %}query{% else %}answer{% endif %} from {{ dm.NetworkInfo.QueryIP }}#{{ dm.NetworkInfo.QueryPort }}:
+;; ->>HEADER<<- opcode: {{ dm.DNS.Opcode }}, status: {{ dm.DNS.Rcode }}, id: {{ dm.DNS.ID }}
+;; flags: {{ dm.DNS.Flags.QR | yesno:"qr ," }}{{ dm.DNS.Flags.RD | yesno:"rd ," }}{{ dm.DNS.Flags.RA | yesno:"ra ," }}; QUERY: {{ dm.DNS.QuestionsCount }}, ANSWER: {{ dm.DNS.DNSRRs.Answers | length }}, AUTHORITY: {{ dm.DNS.DNSRRs.Nameservers | length }}, ADDITIONAL: {{ dm.DNS.DNSRRs.Records | length }}
+
+;; QUESTION SECTION:
+;{{ dm.DNS.Qname }}		{{ dm.DNS.Qclass }}	{{ dm.DNS.Qtype }}
+
+;; ANSWER SECTION: {% for rr in dm.DNS.DNSRRs.Answers %}
+{{ rr.Name }}		{{ rr.TTL }} {{ rr.Class }} {{ rr.Rdatatype }} {{ rr.Rdata }}{% endfor %}
+
+;; WHEN: {{ dm.DNSTap.Timestamp }}
+;; MSG SIZE  rcvd: {{ dm.DNS.Length }}`
+
+	text, err := dm.ToTextTemplate(template)
+	if err != nil {
+		t.Errorf("Want no error, got: %s", err)
+	}
+
+	if !strings.Contains(text, dm.DNS.Qname) {
+		t.Errorf("Want qname in template, got: %s", text)
+	}
+}
+
+func BenchmarkDnsMessage_ToJinjaFormat(b *testing.B) {
+	dm := DNSMessage{}
+	dm.Init()
+	dm.InitTransforms()
+
+	template := `
+;; Got {% if dm.DNS.Type == "QUERY" %}query{% else %}answer{% endif %} from {{ dm.NetworkInfo.QueryIP }}#{{ dm.NetworkInfo.QueryPort }}:
+;; ->>HEADER<<- opcode: {{ dm.DNS.Opcode }}, status: {{ dm.DNS.Rcode }}, id: {{ dm.DNS.ID }}
+;; flags: {{ dm.DNS.Flags.QR | yesno:"qr ," }}{{ dm.DNS.Flags.RD | yesno:"rd ," }}{{ dm.DNS.Flags.RA | yesno:"ra ," }}; QUERY: {{ dm.DNS.QuestionsCount }}, ANSWER: {{ dm.DNS.DNSRRs.Answers | length }}, AUTHORITY: {{ dm.DNS.DNSRRs.Nameservers | length }}, ADDITIONAL: {{ dm.DNS.DNSRRs.Records | length }}
+
+;; QUESTION SECTION:
+;{{ dm.DNS.Qname }}		{{ dm.DNS.Qclass }}	{{ dm.DNS.Qtype }}
+
+;; ANSWER SECTION: {% for rr in dm.DNS.DNSRRs.Answers %}
+{{ rr.Name }}		{{ rr.TTL }} {{ rr.Class }} {{ rr.Rdatatype }} {{ rr.Rdata }}{% endfor %}
+
+;; WHEN: {{ dm.DNSTap.Timestamp }}
+;; MSG SIZE  rcvd: {{ dm.DNS.Length }}`
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := dm.ToTextTemplate(template)
+		if err != nil {
+			b.Fatalf("could not encode to template: %v\n", err)
+		}
+	}
+}

docs/configuration.md

@@ -123,13 +123,11 @@ global:
   text-format-delimiter: " "
   text-format-splitter: " "
   text-format-boundary: "\""
-
+  text-jinja: ""
 ```
 
 If you require a output format like CSV, the delimiter can be configured with the `text-format-delimiter` option.
-The default separator is [space].
-
-text-format can contain raw text enclosed by curly braces, eg
+The default separator is [space]. text-format can contain raw text enclosed by curly braces, eg
 
 ```yaml
 	text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport qname qtype {DNSTAP}"
@@ -145,6 +143,25 @@ Output example:
 
 ```
 
+If you want a more flexible format, you can use the `text-jinja` setting
+Example to enable output similiar to dig style:
+
+```
+text-jinja: |+
+    ;; Got {% if dm.DNS.Type == "QUERY" %}query{% else %}answer{% endif %} from {{ dm.NetworkInfo.QueryIP }}#{{ dm.NetworkInfo.QueryPort }}:
+    ;; ->>HEADER<<- opcode: {{ dm.DNS.Opcode }}, status: {{ dm.DNS.Rcode }}, id: {{ dm.DNS.ID }}
+    ;; flags: {{ dm.DNS.Flags.QR | yesno:"qr ," }}{{ dm.DNS.Flags.RD | yesno:"rd ," }}{{ dm.DNS.Flags.RA | yesno:"ra ," }}; QUERY: {{ dm.DNS.QuestionsCount }}, ANSWER: {{ dm.DNS.DNSRRs.Answers | length }}, AUTHORITY: {{ dm.DNS.DNSRRs.Nameservers | length }}, ADDITIONAL: {{ dm.DNS.DNSRRs.Records | length }}
+    
+    ;; QUESTION SECTION:
+    ;{{ dm.DNS.Qname }}		{{ dm.DNS.Qclass }}	{{ dm.DNS.Qtype }}
+
+    ;; ANSWER SECTION: {% for rr in dm.DNS.DNSRRs.Answers %}
+    {{ rr.Name }}		{{ rr.TTL }} {{ rr.Class }} {{ rr.Rdatatype }} {{ rr.Rdata }}{% endfor %}
+
+    ;; WHEN: {{ dm.DNSTap.Timestamp }}
+    ;; MSG SIZE  rcvd: {{ dm.DNS.Length }}
+```
+
 ## Pid file
 
 Set path to create DNS-collector PID.

docs/loggers/logger_file.md

@@ -3,7 +3,7 @@
 Enable this logger if you want to log your DNS traffic to a file in plain text mode or binary mode.
 
 * with rotation file support
-* supported format: `text`, `json` and `flat json`, `pcap` or `dnstap`
+* supported format: `text`, `jinja`, `json` and `flat json`, `pcap` or `dnstap`
 * gzip compression
 * execute external command after each rotation
 * custom text format
@@ -39,7 +39,7 @@ Options:
   > run external script after file compress step
 
 * `mode` (string)
-  > output format: text, json, flat-json, pcap or dnstap
+  > output format: text, jinja, json, flat-json, pcap or dnstap
 
 * `text-format` (string)
   > output text format, please refer to the default text format to see all

docs/loggers/logger_stdout.md

@@ -3,13 +3,13 @@
 Print to your standard output, all DNS logs received
 
 * in text or json format
-* custom text format
+* custom text format (with jinja templating support)
 * binary mode (pcap)
 
 Options:
 
 * `mode` (string)
-  > output format: `text`, `json`, `flat-json` or `pcap`
+  > output format: `text`, `jinja`, `json`, `flat-json` or `pcap`
 
 * `text-format` (string)
   > output text format, please refer to the default text format to see all available [directives](../configuration.md#custom-text-format), use this parameter if you want a specific format

docs/performance.md

@@ -18,13 +18,15 @@ goos: linux
 goarch: amd64
 pkg: github.com/dmachard/go-dnscollector/dnsutils
 cpu: Intel(R) Core(TM) i5-7200U CPU @ 2.50GHz 
-BenchmarkDnsMessage_ToTextFormat-4               2555529             450.2 ns/op              80 B/op          4 allocs/op
-BenchmarkDnsMessage_ToPacketLayer-4              1138892             952.0 ns/op            1144 B/op         12 allocs/op
-BenchmarkDnsMessage_ToDNSTap-4                   1036468              1136 ns/op             592 B/op         18 allocs/op
-BenchmarkDnsMessage_ToExtendedDNSTap-4            612438              1970 ns/op            1056 B/op         25 allocs/op
-BenchmarkDnsMessage_ToJSON-4                      188379              6724 ns/op            3632 B/op          3 allocs/op
-BenchmarkDnsMessage_ToFlatten-4                   121525             10151 ns/op            8215 B/op         29 allocs/op
-BenchmarkDnsMessage_ToFlatJSON-4                   20704             58365 ns/op           22104 B/op        220 allocs/op
+BenchmarkDnsMessage_ToTextFormat-4               2262946               518.8 ns/op            80 B/op          4 allocs/op
+BenchmarkDnsMessage_ToPacketLayer-4              1241736               926.9 ns/op          1144 B/op         12 allocs/op
+BenchmarkDnsMessage_ToDNSTap-4                    894579              1464 ns/op             592 B/op         18 allocs/op
+BenchmarkDnsMessage_ToExtendedDNSTap-4            608203              2342 ns/op            1056 B/op         25 allocs/op
+BenchmarkDnsMessage_ToJSON-4                      130080              7749 ns/op            3632 B/op          3 allocs/op
+BenchmarkDnsMessage_ToFlatten-4                   117115              9227 ns/op            8369 B/op         29 allocs/op
+BenchmarkDnsMessage_ToFlatJSON-4                   21238             54535 ns/op           20106 B/op        219 allocs/op
+BenchmarkDnsMessage_ToFlatten_Relabelling-4        35614             32544 ns/op            8454 B/op         30 allocs/op
+BenchmarkDnsMessage_ToJinjaFormat-4                 9840            120301 ns/op           50093 B/op        959 allocs/op
 ```
 
 ## Memory usage

go.mod

@@ -72,6 +72,7 @@ require (
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/flosch/pongo2/v6 v6.0.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect

go.sum

@@ -204,6 +204,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU=
+github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=

pkgconfig/constants.go

@@ -18,6 +18,7 @@ const (
 	BadVeryLongDomain = "ultramegaverytoolonglabel.dnscollector" + badLongLabel + badLongLabel +
 		badLongLabel + badLongLabel + badLongLabel + ".dev."
 
+	ModeJinja    = "jinja"
 	ModeText     = "text"
 	ModeJSON     = "json"
 	ModeFlatJSON = "flat-json"

pkgconfig/global.go

@@ -9,8 +9,8 @@ import (
 type ConfigGlobal struct {
 	TextFormat          string `yaml:"text-format" default:"timestamp identity operation rcode queryip queryport family protocol length-unit qname qtype latency"`
 	TextFormatDelimiter string `yaml:"text-format-delimiter" default:" "`
-	TextFormatSplitter  string `yaml:"text-format-splitter" default:" "`
 	TextFormatBoundary  string `yaml:"text-format-boundary" default:"\""`
+	TextJinja           string `yaml:"text-jinja" default:""`
 	Trace               struct {
 		Verbose      bool   `yaml:"verbose" default:"false"`
 		LogMalformed bool   `yaml:"log-malformed" default:"false"`

workers/dnsprocessor.go

@@ -61,6 +61,9 @@ func (w *DNSProcessor) StartCollect() {
 				w.LogError("dns parser malformed packet: %s - %v+", err, dm)
 			}
 
+			// get number of questions
+			dm.DNS.QuestionsCount = dnsHeader.Qdcount
+
 			// dns reply ?
 			if dnsHeader.Qr == 1 {
 				dm.DNSTap.Operation = "CLIENT_RESPONSE"

workers/dnstapserver.go

@@ -512,6 +512,9 @@ func (w *DNSTapProcessor) StartCollect() {
 					}
 				}
 
+				// get number of questions
+				dm.DNS.QuestionsCount = dnsHeader.Qdcount
+
 				if err = dnsutils.DecodePayload(&dm, &dnsHeader, w.GetConfig()); err != nil {
 					dm.DNS.MalformedPacket = true
 					w.LogInfo("dns payload parser stopped: %s", err)

workers/logfile.go

@@ -34,6 +34,7 @@ const (
 func IsValid(mode string) bool {
 	switch mode {
 	case
+		pkgconfig.ModeJinja,
 		pkgconfig.ModeText,
 		pkgconfig.ModeJSON,
 		pkgconfig.ModeFlatJSON,
@@ -506,11 +507,21 @@ func (w *LogFile) StartLogging() {
 				delimiter.WriteString("\n")
 				w.WriteToPlain(delimiter.Bytes())
 
+			// with custom text mode
+			case pkgconfig.ModeJinja:
+				textLine, err := dm.ToTextTemplate(w.GetConfig().Global.TextJinja)
+				if err != nil {
+					w.LogError("jinja template: %s", err)
+					continue
+				}
+				w.WriteToPlain([]byte(textLine))
+
 			// with json mode
 			case pkgconfig.ModeFlatJSON:
 				flat, err := dm.Flatten()
 				if err != nil {
 					w.LogError("flattening DNS message failed: %e", err)
+					continue
 				}
 				json.NewEncoder(buffer).Encode(flat)
 				w.WriteToPlain(buffer.Bytes())

workers/stdout.go

@@ -20,6 +20,7 @@ import (
 func IsStdoutValidMode(mode string) bool {
 	switch mode {
 	case
+		pkgconfig.ModeJinja,
 		pkgconfig.ModeText,
 		pkgconfig.ModeJSON,
 		pkgconfig.ModeFlatJSON,
@@ -186,6 +187,14 @@ func (w *StdOut) StartLogging() {
 			case pkgconfig.ModeText:
 				w.writerText.Print(dm.String(w.textFormat, w.GetConfig().Global.TextFormatDelimiter, w.GetConfig().Global.TextFormatBoundary))
 
+			case pkgconfig.ModeJinja:
+				textLine, err := dm.ToTextTemplate(w.GetConfig().Global.TextJinja)
+				if err != nil {
+					w.LogError("process: unable to update template: %s", err)
+					continue
+				}
+				w.writerText.Print(textLine)
+
 			case pkgconfig.ModeJSON:
 				json.NewEncoder(buffer).Encode(dm)
 				w.writerText.Print(buffer.String())