Skip to content

Commit b09705c

Browse files
vparfonovvparfonov
committed
LOG-6863: safety parsing/read fields to avoid messing data in log event
Signed-off-by: Vitalii Parfonov <[email protected]>
1 parent b00c77d commit b09705c

File tree

9 files changed

+106
-68
lines changed

9 files changed

+106
-68
lines changed

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -216,11 +216,11 @@ test-env: ## Echo test environment, useful for running tests outside of the Make
216216
RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \
217217

218218
.PHONY: test-functional
219-
test-functional: test-functional-benchmarker-vector
219+
test-functional:
220220
RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \
221221
RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \
222222
go test -race \
223-
./test/functional/... \
223+
./test/functional/outputs/syslog/... \
224224
-ginkgo.noColor -timeout=40m -ginkgo.slowSpecThreshold=45.0
225225

226226
.PHONY: test-forwarder-generator

‎internal/generator/vector/output/common/template/template.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,12 @@ func TemplateRemap(componentID string, inputs []string, userTemplate, field, des
4747

4848
// TransformUserTemplateToVRL converts the user entered template to VRL compatible syntax
4949
// Example: foo-{.log_type||"none"} -> "foo-" + to_string!(.log_type||"none")
50-
func TransformUserTemplateToVRL(userTemplate string) string {
50+
func TransformUserTemplateToVRL(userTemplate string, suffix ...string) string {
5151
// Finds and replaces expressions defined in `{}` with to_string!()
5252
replacedUserTemplate := ReplaceBracketWithToString(userTemplate, "to_string!(%s)")
53+
if len(suffix) > 0 && suffix[0] != "" {
54+
replacedUserTemplate = ReplaceBracketWithToString(userTemplate, "to_string!("+suffix[0]+"%s)")
55+
}
5356

5457
// Finding all matches of to_string!() returning their start + end indices
5558
matchedIndices := splitRegex.FindAllStringSubmatchIndex(replacedUserTemplate, -1)

‎internal/generator/vector/output/syslog/rfc3164_with_defaults.toml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
type = "remap"
33
inputs = ["application"]
44
source = '''
5-
. = merge(., parse_json!(string!(.message))) ?? .
5+
#calculate defaults
66
if .log_type == "infrastructure" && .log_source == "node" {
77
._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")
88
._internal.syslog.proc_id = to_string!(.systemd.t.PID || "")
@@ -21,6 +21,14 @@ if .log_type == "audit" {
2121
._internal.syslog.severity = "informational"
2222
._internal.syslog.facility = "security"
2323
}
24+
_tmp, err = parse_json(string!(.message))
25+
if err != null {
26+
_tmp = .
27+
log(err, level: "error")
28+
} else {
29+
_tmp = merge!(.,_tmp)
30+
}
31+
parsed_msg = _tmp
2432
.facility = to_string!(._internal.syslog.facility || "user")
2533
.severity = to_string!(._internal.syslog.severity || "informational")
2634
.proc_id = to_string!(._internal.syslog.proc_id || "-")

‎internal/generator/vector/output/syslog/syslog.go

Lines changed: 34 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ import (
2020
)
2121

2222
const (
23-
TCP = `tcp`
24-
TLS = `tls`
23+
TCP = `tcp`
24+
TLS = `tls`
25+
ParsedMsg = "parsed_msg"
2526
)
2627

2728
type Syslog struct {
@@ -68,13 +69,12 @@ func (ser SyslogEncodingRemap) Name() string {
6869
}
6970

7071
func (ser SyslogEncodingRemap) Template() string {
71-
return `{{define "` + ser.Name() + `" -}}
72+
return fmt.Sprintf(`{{define "`+ser.Name()+`" -}}
7273
[transforms.{{.ComponentID}}]
7374
type = "remap"
7475
inputs = {{.Inputs}}
7576
source = '''
76-
. = merge(., parse_json!(string!(.message))) ?? .
77-
77+
#calculate defaults
7878
{{if eq .RFC "RFC3164" -}}
7979
if .log_type == "infrastructure" && .log_source == "node" {
8080
._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")
@@ -118,7 +118,16 @@ if .log_type == "audit" {
118118
}
119119
{{end}}
120120
121+
121122
{{if .EncodingFields.FieldVRLList -}}
123+
_tmp, err = parse_json(string!(.message))
124+
if err != null {
125+
_tmp = .
126+
log(err, level: "error")
127+
} else {
128+
_tmp = merge!(.,_tmp)
129+
}
130+
%s = _tmp
122131
{{range $templatePair := .EncodingFields.FieldVRLList -}}
123132
.{{$templatePair.Field}} = {{$templatePair.VRLString}}
124133
{{end -}}
@@ -139,7 +148,7 @@ if is_null({{.PayloadKey}}) {
139148
{{end}}
140149
'''
141150
{{end -}}
142-
`
151+
`, ParsedMsg)
143152
}
144153

145154
type SyslogEncoding struct {
@@ -219,55 +228,30 @@ func getEncodingTemplatesAndFields(s obs.Syslog) EncodingTemplateField {
219228
FieldVRLList: []FieldVRLStringPair{},
220229
}
221230

222-
if s.Facility == "" {
223-
s.Facility = `{._internal.syslog.facility || "user"}`
231+
appendField := func(fieldName string, value *string, defaultVal string) {
232+
if *value == "" {
233+
*value = defaultVal
234+
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
235+
Field: fieldName,
236+
VRLString: commontemplate.TransformUserTemplateToVRL(*value),
237+
})
238+
} else {
239+
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
240+
Field: fieldName,
241+
VRLString: commontemplate.TransformUserTemplateToVRL(*value, ParsedMsg),
242+
})
243+
}
224244
}
225-
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
226-
Field: "facility",
227-
VRLString: commontemplate.TransformUserTemplateToVRL(s.Facility),
228-
})
229245

230-
if s.Severity == "" {
231-
s.Severity = `{._internal.syslog.severity || "informational"}`
232-
}
233-
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
234-
Field: "severity",
235-
VRLString: commontemplate.TransformUserTemplateToVRL(s.Severity),
236-
})
237-
238-
if s.ProcId == "" {
239-
s.ProcId = `{._internal.syslog.proc_id || "-"}`
240-
}
241-
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
242-
Field: "proc_id",
243-
VRLString: commontemplate.TransformUserTemplateToVRL(s.ProcId),
244-
})
246+
appendField("facility", &s.Facility, `{._internal.syslog.facility || "user"}`)
247+
appendField("severity", &s.Severity, `{._internal.syslog.severity || "informational"}`)
248+
appendField("proc_id", &s.ProcId, `{._internal.syslog.proc_id || "-"}`)
245249

246250
if s.RFC == obs.SyslogRFC3164 {
247-
if s.AppName == "" {
248-
s.AppName = `{._internal.syslog.tag || ""}`
249-
}
250-
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
251-
Field: "tag",
252-
VRLString: commontemplate.TransformUserTemplateToVRL(s.AppName),
253-
})
254-
251+
appendField("tag", &s.AppName, `{._internal.syslog.tag || ""}`)
255252
} else {
256-
if s.AppName == "" {
257-
s.AppName = `{._internal.syslog.app_name || "-"}`
258-
}
259-
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
260-
Field: "app_name",
261-
VRLString: commontemplate.TransformUserTemplateToVRL(s.AppName),
262-
})
263-
264-
if s.MsgId == "" {
265-
s.MsgId = `{._internal.syslog.msg_id || "-"}`
266-
}
267-
templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{
268-
Field: "msg_id",
269-
VRLString: commontemplate.TransformUserTemplateToVRL(s.MsgId),
270-
})
253+
appendField("app_name", &s.AppName, `{._internal.syslog.app_name || "-"}`)
254+
appendField("msg_id", &s.MsgId, `{._internal.syslog.msg_id || "-"}`)
271255
}
272256

273257
return templateFields

‎internal/generator/vector/output/syslog/tcp_with_defaults.toml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
type = "remap"
33
inputs = ["application"]
44
source = '''
5-
. = merge(., parse_json!(string!(.message))) ?? .
5+
#calculate defaults
66
._internal.syslog.msg_id = .log_source
77
88
if .log_type == "infrastructure" && .log_source == "node" {
@@ -21,6 +21,15 @@ if .log_type == "audit" {
2121
._internal.syslog.severity = "informational"
2222
._internal.syslog.facility = "security"
2323
}
24+
_tmp, err = parse_json(string!(.message))
25+
if err != null {
26+
_tmp = .
27+
log(err, level: "error")
28+
} else {
29+
_tmp = merge!(.,_tmp)
30+
}
31+
parsed_msg = _tmp
32+
2433
.facility = to_string!(._internal.syslog.facility || "user")
2534
.severity = to_string!(._internal.syslog.severity || "informational")
2635
.proc_id = to_string!(._internal.syslog.proc_id || "-")

‎internal/generator/vector/output/syslog/tcp_with_tuning.toml

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,8 @@
22
type = "remap"
33
inputs = ["application"]
44
source = '''
5-
. = merge(., parse_json!(string!(.message))) ?? .
6-
5+
#calculate defaults
76
._internal.syslog.msg_id = .log_source
8-
97
if .log_type == "infrastructure" && .log_source == "node" {
108
._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")
119
._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-")
@@ -22,6 +20,15 @@ if .log_type == "audit" {
2220
._internal.syslog.severity = "informational"
2321
._internal.syslog.facility = "security"
2422
}
23+
_tmp, err = parse_json(string!(.message))
24+
if err != null {
25+
_tmp = .
26+
log(err, level: "error")
27+
} else {
28+
_tmp = merge!(.,_tmp)
29+
}
30+
parsed_msg = _tmp
31+
2532
.facility = to_string!(._internal.syslog.facility || "user")
2633
.severity = to_string!(._internal.syslog.severity || "informational")
2734
.proc_id = to_string!(._internal.syslog.proc_id || "-")

‎internal/generator/vector/output/syslog/tls_with_field_references.toml

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,8 @@
22
type = "remap"
33
inputs = ["application"]
44
source = '''
5-
. = merge(., parse_json!(string!(.message))) ?? .
6-
5+
#calculate defaults
76
._internal.syslog.msg_id = .log_source
8-
97
if .log_type == "infrastructure" && .log_source == "node" {
108
._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")
119
._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-")
@@ -22,11 +20,21 @@ if .log_type == "audit" {
2220
._internal.syslog.severity = "informational"
2321
._internal.syslog.facility = "security"
2422
}
25-
.facility = to_string!(.facility||"none")
26-
.severity = to_string!(.severity||"none")
27-
.proc_id = to_string!(.proc_id||"none")
28-
.app_name = to_string!(.app_name||"none")
29-
.msg_id = to_string!(.msg_id||"none")
23+
24+
_tmp, err = parse_json(string!(.message))
25+
if err != null {
26+
_tmp = .
27+
log(err, level: "error")
28+
} else {
29+
_tmp = merge!(.,_tmp)
30+
}
31+
parsed_msg = _tmp
32+
33+
.facility = to_string!(parsed_msg.facility||"none")
34+
.severity = to_string!(parsed_msg.severity||"none")
35+
.proc_id = to_string!(parsed_msg.proc_id||"none")
36+
.app_name = to_string!(parsed_msg.app_name||"none")
37+
.msg_id = to_string!(parsed_msg.msg_id||"none")
3038
3139
if is_null(.payload_key) {
3240
.payload_key = .

‎internal/generator/vector/output/syslog/udp_with_every_setting.toml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
type = "remap"
33
inputs = ["application"]
44
source = '''
5-
. = merge(., parse_json!(string!(.message))) ?? .
5+
#calculate defaults
66
if .log_type == "infrastructure" && .log_source == "node" {
77
._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")
88
._internal.syslog.proc_id = to_string!(.systemd.t.PID || "")
@@ -21,6 +21,16 @@ if .log_type == "audit" {
2121
._internal.syslog.severity = "informational"
2222
._internal.syslog.facility = "security"
2323
}
24+
25+
_tmp, err = parse_json(string!(.message))
26+
if err != null {
27+
_tmp = .
28+
log(err, level: "error")
29+
} else {
30+
_tmp = merge!(.,_tmp)
31+
}
32+
parsed_msg = _tmp
33+
2434
.facility = "kern"
2535
.severity = "critical"
2636
.proc_id = "procID"

‎internal/generator/vector/output/syslog/xyz_defaults.toml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
type = "remap"
33
inputs = ["application"]
44
source = '''
5-
. = merge(., parse_json!(string!(.message))) ?? .
5+
#calculate defaults
66
._internal.syslog.msg_id = .log_source
77
if .log_type == "infrastructure" && .log_source == "node" {
88
._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")
@@ -21,6 +21,15 @@ if .log_type == "audit" {
2121
._internal.syslog.facility = "security"
2222
}
2323
24+
_tmp, err = parse_json(string!(.message))
25+
if err != null {
26+
_tmp = .
27+
log(err, level: "error")
28+
} else {
29+
_tmp = merge!(.,_tmp)
30+
}
31+
parsed_msg = _tmp
32+
2433
.facility = to_string!(._internal.syslog.facility || "user")
2534
.severity = to_string!(._internal.syslog.severity || "informational")
2635
.proc_id = to_string!(._internal.syslog.proc_id || "-")

0 commit comments

Comments
 (0)