Merge pull request #91 from cloud-gov/logstash-trim

Logstash trim

jobs/opensearch/templates/config/jvm.options.erb

@@ -11,7 +11,7 @@
 ## -Xms4g
 ## -Xmx4g
 ##
-## See https://www.open.co/guide/en/opensearch/reference/current/heap-size.html
+## See https://opensearch.org/docs/opensearch/install/important-settings/
 ## for more information
 ##
 ################################################################
@@ -22,8 +22,6 @@
 #-Xms2g
 #-Xmx2g
 
--XX:+DisableExplicitGC
-
 ################################################################
 ## Expert settings
 ################################################################
@@ -34,6 +32,50 @@
 ##
 ################################################################
 
+## GC configuration
+8-10:-XX:+UseConcMarkSweepGC
+8-10:-XX:CMSInitiatingOccupancyFraction=75
+8-10:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1GC is the default GC for all JDKs 11 and newer
+11-:-XX:+UseG1GC
+# See https://github.com/elastic/elasticsearch/pull/46169 for the history
+# behind these settings, but the tl;dr is that default values can lead
+# to situations where heap usage grows enough to trigger a circuit breaker
+# before GC kicks in.
+11-:-XX:G1ReservePercent=25
+11-:-XX:InitiatingHeapOccupancyPercent=30
+
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+## JDK 8 GC logging
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:${loggc}
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=${loggc}:utctime,pid,tags:filecount=32,filesize=64m
+
+# Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380)
+18-:-Djava.security.manager=allow
+
+# JDK 20+ Incubating Vector Module for SIMD optimizations;
+# disabling may reduce performance on vector optimized lucene
+20-:--add-modules=jdk.incubator.vector
+
+# HDFS ForkJoinPool.common() support by SecurityManager
+-Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory
 
 ## optimizations
 

src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf

@@ -68,15 +68,9 @@ if [@source][type] =~ /APP(|\/.*)$/ {
 
     } else {
 
-        ## ---- Format 3: Logback status logs
-        grok {
-            match => [ "@message", "%{TIME} \|\-%{LOGLEVEL:@level} in %{NOTSPACE:[app][logger]} - %{GREEDYDATA:@message}" ]
-            overwrite => [ "@message", "@level" ]
-
-            ## ---- Unknown Format: if no formats succeeded set with 'unknown_msg_format' tag
-            tag_on_failure => [ "unknown_msg_format" ]
-            id => "cloudfoundry/app-app/logback/grok"
-        }
+        mutate {
+                add_tag => [ "unknown_msg_format" ]
+            }
     }
 
 }

src/cf-logstash-filters/src/logstash-filters/snippets/setup.conf

@@ -24,6 +24,10 @@ if [syslog_sd_params][app_id] {
     update => { "@index_type" => "app"}
   }
 }
+else
+{
+  drop { }
+}
 
 
 mutate {

src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf

@@ -68,15 +68,9 @@ if [@type] == "LogMessage" and [@source][type] =~ /APP(|\/.*)$/ {
 
     } else {
 
-        ## ---- Format 3: Logback status logs
-        grok {
-            match => [ "@message", "%{TIME} \|\-%{LOGLEVEL:@level} in %{NOTSPACE:[app][logger]} - %{GREEDYDATA:@message}" ]
-            overwrite => [ "@message", "@level" ]
-
-            ## ---- Unknown Format: if no formats succeeded set with 'unknown_msg_format' tag
-            tag_on_failure => [ "unknown_msg_format" ]
-            id => "cloudfoundry/app-app/logback/grok"
-        }
+        mutate {
+                add_tag => [ "unknown_msg_format" ]
+            }
     }
 
 }