Merge pull request #119 from cloud-gov/trimmed

updating to truncate only on message
cloud-gov · Nov 25, 2024 · d053759 · d053759
2 parents 05f98e2 + cc7fbef
commit d053759
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 37 deletions.
diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec
@@ -149,7 +149,7 @@ properties:
     default: false
   logstash_parser.message_max_size:
     description: "Maximum log message length.  Anything larger is truncated (TODO: move this to ingestor?)"
-    default: 32765
+    default: 1048576
   logstash_parser.filters:
     description: "The configuration to embed into the logstash filters section. Can either be a set of parsing rules as a string or a list of hashes in the form of [{name: path_to_parsing_rules.conf}]"
     default: ""

diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb
@@ -106,18 +106,10 @@
     #
     # trim excessively long messages
     #
-
-    ruby {
-        code => '
-            max_line_length = <%= p("logstash_parser.message_max_size") %>
-            message = event.get("@message")
-            if message && message.length > max_line_length
-                event.set("@message", message[0, max_line_length])
-                tags = event.get("tags")
-                tags ||= [] << "_groktrimmed"
-                event.set("tags", tags)
-            end
-        '
+    truncate {
+            fields => ["@message"]
+            add_tag => [ "_logtrimmed" ]
+            length_bytes => <%= p("logstash_parser.message_max_size") %>
     }
 
     #

diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf
@@ -18,37 +18,43 @@ if [@source][type] =~ /APP(|\/.*)$/ {
 
     ## ---- Format 1: JSON
     if [@message] =~ /^\s*{".*}\s*$/ { # if it looks like JSON
-
-        json {
-            source => "@message"
-            target => "app"
-            id => "cloudfoundry/app-message/json"
+        truncate {
+            fields => ["@message"]
+            add_tag => [ "_messagetrimmed" ]
+            length_bytes => 32765
         }
+        if !("_messagetrimmed" in [tags]) {
+            json {
+                source => "@message"
+                target => "app"
+                id => "cloudfoundry/app-message/json"
+            }
 
-        if !("_jsonparsefailure" in [tags]) {
+            if !("_jsonparsefailure" in [tags]) {
 
-            mutate {
-                rename => { "[app][message]" => "@message" } # @message
-            }
-            # concat message and exception
-            if [app][exception] {
                 mutate {
-                    ## NOTE: keep line break and new line spacing (new line is inserted in logstash in such a way)
-                    replace => { "@message" => "%{@message}
-%{[app][exception]}" }
-                    remove_field => [ "[app][exception]" ]
-              }
-            }
+                    rename => { "[app][message]" => "@message" } # @message
+                }
+                # concat message and exception
+                if [app][exception] {
+                    mutate {
+                        ## NOTE: keep line break and new line spacing (new line is inserted in logstash in such a way)
+                        replace => { "@message" => "%{@message}
+    %{[app][exception]}" }
+                        remove_field => [ "[app][exception]" ]
+                    }
+                }
 
-            mutate {
-                rename => { "[app][level]" => "@level" } # @level
-            }
+                mutate {
+                    rename => { "[app][level]" => "@level" } # @level
+                }
 
-        } else {
+            } else {
 
-            mutate {
-                add_tag => [ "unknown_msg_format" ]
-                remove_tag => ["_jsonparsefailure"]
+                mutate {
+                    add_tag => [ "unknown_msg_format" ]
+                    remove_tag => ["_jsonparsefailure"]
+                }
             }
         }