From 89fdf47672507d24a42565cabcc56688048a4f07 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 11 Oct 2024 10:16:29 -0400 Subject: [PATCH 1/6] This tag will be applied either way so no use in doing a regex to see, if its not an app log than we don't need to continue with it --- .../snippets/app-logmessage-app.conf | 12 +++--------- .../src/logstash-filters/snippets/setup.conf | 4 ++++ .../snippets/app-logmessage-app.conf | 12 +++--------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf index 63ec5a4c..d3261df5 100644 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf +++ b/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-app.conf @@ -68,15 +68,9 @@ if [@source][type] =~ /APP(|\/.*)$/ { } else { - ## ---- Format 3: Logback status logs - grok { - match => [ "@message", "%{TIME} \|\-%{LOGLEVEL:@level} in %{NOTSPACE:[app][logger]} - %{GREEDYDATA:@message}" ] - overwrite => [ "@message", "@level" ] - - ## ---- Unknown Format: if no formats succeeded set with 'unknown_msg_format' tag - tag_on_failure => [ "unknown_msg_format" ] - id => "cloudfoundry/app-app/logback/grok" - } + mutate { + add_tag => [ "unknown_msg_format" ] + } } } diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/setup.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/setup.conf index fdf49c9d..f336c385 100644 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/setup.conf +++ b/src/cf-logstash-filters/src/logstash-filters/snippets/setup.conf @@ -24,6 +24,10 @@ if [syslog_sd_params][app_id] { update => { "@index_type" => "app"} } } +else +{ + drop { } +} mutate { diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf index a699c4c8..2e9c8f91 100644 --- a/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-app.conf @@ -68,15 +68,9 @@ if [@type] == "LogMessage" and [@source][type] =~ /APP(|\/.*)$/ { } else { - ## ---- Format 3: Logback status logs - grok { - match => [ "@message", "%{TIME} \|\-%{LOGLEVEL:@level} in %{NOTSPACE:[app][logger]} - %{GREEDYDATA:@message}" ] - overwrite => [ "@message", "@level" ] - - ## ---- Unknown Format: if no formats succeeded set with 'unknown_msg_format' tag - tag_on_failure => [ "unknown_msg_format" ] - id => "cloudfoundry/app-app/logback/grok" - } + mutate { + add_tag => [ "unknown_msg_format" ] + } } } From ccb8ac788d1132c62966598308cb2e655dc0acb9 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 11 Oct 2024 13:46:21 -0400 Subject: [PATCH 2/6] bringing back the gc settings --- jobs/opensearch/templates/config/jvm.options.erb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/jobs/opensearch/templates/config/jvm.options.erb b/jobs/opensearch/templates/config/jvm.options.erb index 33d4be4f..14e3cc03 100644 --- a/jobs/opensearch/templates/config/jvm.options.erb +++ b/jobs/opensearch/templates/config/jvm.options.erb @@ -22,8 +22,6 @@ #-Xms2g #-Xmx2g --XX:+DisableExplicitGC - ################################################################ ## Expert settings ################################################################ @@ -34,6 +32,20 @@ ## ################################################################ +## GC configuration +8-10:-XX:+UseConcMarkSweepGC +8-10:-XX:CMSInitiatingOccupancyFraction=75 +8-10:-XX:+UseCMSInitiatingOccupancyOnly + +## G1GC Configuration +# NOTE: G1GC is the default GC for all JDKs 11 and newer +11-:-XX:+UseG1GC +# See https://github.com/elastic/elasticsearch/pull/46169 for the history +# behind these settings, but the tl;dr is that default values can lead +# to situations where heap usage grows enough to trigger a circuit breaker +# before GC kicks in. +11-:-XX:G1ReservePercent=25 +11-:-XX:InitiatingHeapOccupancyPercent=30 ## optimizations From 5dcf39ef7cfe2dc8c7c9558c5b375f184479ee54 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 11 Oct 2024 13:47:58 -0400 Subject: [PATCH 3/6] still useful --- jobs/opensearch/templates/config/jvm.options.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/opensearch/templates/config/jvm.options.erb b/jobs/opensearch/templates/config/jvm.options.erb index 14e3cc03..f6a7f7dc 100644 --- a/jobs/opensearch/templates/config/jvm.options.erb +++ b/jobs/opensearch/templates/config/jvm.options.erb @@ -32,6 +32,8 @@ ## ################################################################ +XX:+DisableExplicitGC + ## GC configuration 8-10:-XX:+UseConcMarkSweepGC 8-10:-XX:CMSInitiatingOccupancyFraction=75 From d1092dabe94417e453b719187f7c4f97a2c7b0f5 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 11 Oct 2024 13:49:23 -0400 Subject: [PATCH 4/6] still useful --- jobs/opensearch/templates/config/jvm.options.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/opensearch/templates/config/jvm.options.erb b/jobs/opensearch/templates/config/jvm.options.erb index f6a7f7dc..40bdbad9 100644 --- a/jobs/opensearch/templates/config/jvm.options.erb +++ b/jobs/opensearch/templates/config/jvm.options.erb @@ -22,6 +22,8 @@ #-Xms2g #-Xmx2g +-XX:+DisableExplicitGC + ################################################################ ## Expert settings ################################################################ @@ -32,8 +34,6 @@ ## ################################################################ -XX:+DisableExplicitGC - ## GC configuration 8-10:-XX:+UseConcMarkSweepGC 8-10:-XX:CMSInitiatingOccupancyFraction=75 From 45ddeeeed8ac3471bf9a4c873e3510b07b6dfa5c Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 11 Oct 2024 14:52:08 -0400 Subject: [PATCH 5/6] stealing all java settings from upstream --- .../templates/config/jvm.options.erb | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/jobs/opensearch/templates/config/jvm.options.erb b/jobs/opensearch/templates/config/jvm.options.erb index 40bdbad9..3bee1104 100644 --- a/jobs/opensearch/templates/config/jvm.options.erb +++ b/jobs/opensearch/templates/config/jvm.options.erb @@ -11,7 +11,7 @@ ## -Xms4g ## -Xmx4g ## -## See https://www.open.co/guide/en/opensearch/reference/current/heap-size.html +## See https://opensearch.org/docs/opensearch/install/important-settings/ ## for more information ## ################################################################ @@ -19,10 +19,8 @@ # Xms represents the initial size of total heap space # Xmx represents the maximum size of total heap space -#-Xms2g -#-Xmx2g - --XX:+DisableExplicitGC +-Xms${heap.min} +-Xmx${heap.max} ################################################################ ## Expert settings @@ -49,6 +47,36 @@ 11-:-XX:G1ReservePercent=25 11-:-XX:InitiatingHeapOccupancyPercent=30 + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +## JDK 8 GC logging +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:${loggc} +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m + +# JDK 9+ GC logging +9-:-Xlog:gc*,gc+age=trace,safepoint:file=${loggc}:utctime,pid,tags:filecount=32,filesize=64m + +# Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380) +18-:-Djava.security.manager=allow + +# JDK 20+ Incubating Vector Module for SIMD optimizations; +# disabling may reduce performance on vector optimized lucene +20-:--add-modules=jdk.incubator.vector + +# HDFS ForkJoinPool.common() support by SecurityManager +-Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory + ## optimizations # ensure UTF-8 encoding by default (e.g. filenames) From b1dfc89e5de9b7c712889d3aa652500cafcd5959 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Fri, 11 Oct 2024 14:54:22 -0400 Subject: [PATCH 6/6] stealing all java settings from upstream --- jobs/opensearch/templates/config/jvm.options.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/opensearch/templates/config/jvm.options.erb b/jobs/opensearch/templates/config/jvm.options.erb index 3bee1104..e4319daf 100644 --- a/jobs/opensearch/templates/config/jvm.options.erb +++ b/jobs/opensearch/templates/config/jvm.options.erb @@ -19,8 +19,8 @@ # Xms represents the initial size of total heap space # Xmx represents the maximum size of total heap space --Xms${heap.min} --Xmx${heap.max} +#-Xms2g +#-Xmx2g ################################################################ ## Expert settings