diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec index f2e8aed5..23a12049 100644 --- a/jobs/ingestor_syslog/spec +++ b/jobs/ingestor_syslog/spec @@ -149,7 +149,7 @@ properties: default: false logstash_parser.message_max_size: description: "Maximum log message length. Anything larger is truncated (TODO: move this to ingestor?)" - default: 32765 + default: 1048576 logstash_parser.filters: description: "The configuration to embed into the logstash filters section. Can either be a set of parsing rules as a string or a list of hashes in the form of [{name: path_to_parsing_rules.conf}]" default: "" diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index f72a7ee0..1b216ed8 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -106,18 +106,10 @@ # # trim excessively long messages # - - ruby { - code => ' - max_line_length = <%= p("logstash_parser.message_max_size") %> - message = event.get("@message") - if message && message.length > max_line_length - event.set("@message", message[0, max_line_length]) - tags = event.get("tags") - tags ||= [] << "_groktrimmed" - event.set("tags", tags) - end - ' + truncate { + fields => ["@message"] + add_tag => [ "_logtrimmed" ] + length_bytes => <%= p("logstash_parser.message_max_size") %> } # diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf index c8a30d6b..42095449 100644 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf +++ b/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf @@ -18,37 +18,43 @@ if [@source][type] =~ /APP(|\/.*)$/ { ## ---- Format 1: JSON if [@message] =~ /^\s*{".*}\s*$/ { # if it looks like JSON - - json { - source => "@message" - target => "app" - id => "cloudfoundry/app-message/json" + truncate { + fields => ["@message"] + add_tag => [ "_messagetrimmed" ] + length_bytes => 32765 } + if !("_messagetrimmed" in [tags]) { + json { + source => "@message" + target => "app" + id => "cloudfoundry/app-message/json" + } - if !("_jsonparsefailure" in [tags]) { + if !("_jsonparsefailure" in [tags]) { - mutate { - rename => { "[app][message]" => "@message" } # @message - } - # concat message and exception - if [app][exception] { mutate { - ## NOTE: keep line break and new line spacing (new line is inserted in logstash in such a way) - replace => { "@message" => "%{@message} -%{[app][exception]}" } - remove_field => [ "[app][exception]" ] - } - } + rename => { "[app][message]" => "@message" } # @message + } + # concat message and exception + if [app][exception] { + mutate { + ## NOTE: keep line break and new line spacing (new line is inserted in logstash in such a way) + replace => { "@message" => "%{@message} + %{[app][exception]}" } + remove_field => [ "[app][exception]" ] + } + } - mutate { - rename => { "[app][level]" => "@level" } # @level - } + mutate { + rename => { "[app][level]" => "@level" } # @level + } - } else { + } else { - mutate { - add_tag => [ "unknown_msg_format" ] - remove_tag => ["_jsonparsefailure"] + mutate { + add_tag => [ "unknown_msg_format" ] + remove_tag => ["_jsonparsefailure"] + } } }