From 14cb37e291ee1c002cd595fbb10c3c98327007fe Mon Sep 17 00:00:00 2001
From: nikodemas <47255905+nikodemas@users.noreply.github.com>
Date: Wed, 21 Aug 2024 16:48:17 +0200
Subject: [PATCH 1/2] Update aps logstash for prod

---
 kubernetes/cmsweb/monitoring/logstash.conf | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/kubernetes/cmsweb/monitoring/logstash.conf b/kubernetes/cmsweb/monitoring/logstash.conf
index e87ea4e12..9a0ffffcf 100644
--- a/kubernetes/cmsweb/monitoring/logstash.conf
+++ b/kubernetes/cmsweb/monitoring/logstash.conf
@@ -78,7 +78,7 @@ filter {
   if "aps" in [tags] {
       mutate { replace => { "type" => "aps" } }
       grok {
-        match => { "message" => '\[%{TIMESTAMP_ISO8601:tstamp}\] %{DATA:httpversion} %{NUMBER:code:int} %{WORD:method} %{NOTSPACE:request} \[data: %{NUMBER:bytes_received:int} in %{NUMBER:bytes_sent:int} out\] \[host: %{IPORHOST:frontend}:%{NUMBER:fe_port}\] \[remoteAddr: %{IPORHOST:clientip}:%{NUMBER:clientport:int}\] \[X-Forwarded-For: %{IPORHOST:x_forwarded_ip}:%{NUMBER:x_forwarded_port:int}\] \[X-Forwarded-Host: %{HOSTNAME:x_forwarded_host}\] \[auth: %{DATA:tls} %{DATA:crypto} "%{DATA:dn}" %{DATA:auth_name} %{WORD:auth_protocol}\] \[ref: "%{DATA:cluster}" "%{DATA:client}"\] \[req: %{NUMBER:request_time:float} \(s\) proxy-resp: %{NUMBER:proxy_resp_time:float} \(s\)\]' }
+        match => { "message" => '\[%{TIMESTAMP_ISO8601:tstamp}\] %{DATA:httpversion} %{NUMBER:code:int} %{WORD:method} %{NOTSPACE:request} \[data: %{NUMBER:bytes_received:int} in %{NUMBER:bytes_sent:int} out\] \[host: %{IPORHOST:frontend}(?::%{NUMBER:fe_port})?\] \[remoteAddr: %{IPORHOST:clientip}:%{NUMBER:clientport:int}\] \[X-Forwarded-For: (%{IPORHOST:x_forwarded_ip}:%{NUMBER:x_forwarded_port:int})?\] \[X-Forwarded-Host: (%{HOSTNAME:x_forwarded_host})?\] \[auth: %{DATA:tls} %{DATA:crypto} "%{DATA:dn}" %{DATA:auth_name} %{WORD:auth_protocol}\] \[ref: "%{DATA:cluster}" "%{DATA:client}"\] \[req: %{NUMBER:request_time:float} \(s\) proxy-resp: %{NUMBER:proxy_resp_time:float} \(s\)\]' }
       }
       grok {
         match => {
@@ -124,7 +124,6 @@ filter {
       }
       if ![api] {
           mutate { replace => { "api" => "%{request}" } }
-          mutate { replace => { "system" => "%{request}" } }
       }
       if [client] {
           grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } }
@@ -241,7 +240,7 @@ filter {
   # common filters
 
   # drop failed records
-  if "_grokparsefailure" in [tags] { drop { } }
+  # if "_grokparsefailure" in [tags] { drop { } }
   # remove quotes from message entry since it will break the JSON
   mutate { gsub => [ "message", "\n", "", "message", "\"", ""] }
 

From dd7a1a749ebebaf9af430fd65db4409f071a0ba7 Mon Sep 17 00:00:00 2001
From: nikodemas <47255905+nikodemas@users.noreply.github.com>
Date: Wed, 21 Aug 2024 16:59:56 +0200
Subject: [PATCH 2/2] Remove grokparsefailure part

---
 kubernetes/cmsweb/monitoring/logstash.conf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/kubernetes/cmsweb/monitoring/logstash.conf b/kubernetes/cmsweb/monitoring/logstash.conf
index 9a0ffffcf..fc45256e6 100644
--- a/kubernetes/cmsweb/monitoring/logstash.conf
+++ b/kubernetes/cmsweb/monitoring/logstash.conf
@@ -239,8 +239,6 @@ filter {
 
   # common filters
 
-  # drop failed records
-  # if "_grokparsefailure" in [tags] { drop { } }
   # remove quotes from message entry since it will break the JSON
   mutate { gsub => [ "message", "\n", "", "message", "\"", ""] }