From 63b2fbedf73965842afcc1b323170208c658de91 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Thu, 10 Oct 2024 14:39:28 -0400 Subject: [PATCH 1/2] optimizing logstash --- .../src/logstash-filters/default.conf.erb | 6 +++--- .../src/logstash-filters/default.conf.erb | 12 ++++++------ .../snippets/app-logmessage-rtr.conf | 8 +------- .../src/logstash-filters/default.conf.erb | 6 +++--- .../src/logstash-filters/default.conf.erb | 12 ++++++------ .../snippets/app-logmessage-rtr.conf | 10 ++-------- 6 files changed, 21 insertions(+), 33 deletions(-) diff --git a/src/base-logstash-filters/src/logstash-filters/default.conf.erb b/src/base-logstash-filters/src/logstash-filters/default.conf.erb index 1beaf2dc..24a672ad 100644 --- a/src/base-logstash-filters/src/logstash-filters/default.conf.erb +++ b/src/base-logstash-filters/src/logstash-filters/default.conf.erb @@ -4,6 +4,6 @@ if [@type] in ["syslog", "relp"] { <%= File.read('src/logstash-filters/snippets/syslog_standard.conf').gsub(/^/, ' ') %> } -if [syslog_program] == "nats_to_syslog" { - <%= File.read('src/logstash-filters/snippets/bosh_nats.conf').gsub(/^/, ' ') %> -} +# if [syslog_program] == "nats_to_syslog" { +# <%= File.read('src/logstash-filters/snippets/bosh_nats.conf').gsub(/^/, ' ') %> +# } diff --git a/src/cf-logstash-filters/src/logstash-filters/default.conf.erb b/src/cf-logstash-filters/src/logstash-filters/default.conf.erb index e542ba3c..e7232b6e 100644 --- a/src/cf-logstash-filters/src/logstash-filters/default.conf.erb +++ b/src/cf-logstash-filters/src/logstash-filters/default.conf.erb @@ -19,12 +19,12 @@ ##-- Platform # (Platform snippet should precede all other platform snippets) -<%= File.read('src/logstash-filters/snippets/platform.conf') %> -# special cases parsing -<%= File.read('src/logstash-filters/snippets/platform-haproxy.conf') %> -<%= File.read('src/logstash-filters/snippets/platform-uaa.conf') %> -<%= File.read('src/logstash-filters/snippets/platform-vcap.conf') %> -<%= File.read('src/logstash-filters/snippets/platform-gorouter.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform.conf') %> +# # special cases parsing +# <%= File.read('src/logstash-filters/snippets/platform-haproxy.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform-uaa.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform-vcap.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform-gorouter.conf') %> # Teardown snippet (should follow all other snippets) <%= File.read('src/logstash-filters/snippets/teardown.conf') %> diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf index 17670444..3c5561ff 100644 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf +++ b/src/cf-logstash-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf @@ -12,14 +12,8 @@ if [@source][type] == "RTR" { match => [ "@message", [ # cf-deployment v12.27.0+ "^%{HOSTNAME:[rtr][hostname]} - \[(?%{TIMESTAMP_ISO8601})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"( %{GREEDYDATA:kvpairs})?", - # cf-deployment v12.17.0+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_time:%{NUMBER:[rtr][app_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"%{BASE10NUM:[rtr][app][index]:int|-}\"", # cf-release v252+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"", - # cf-release v250+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{HOSTPORT}|-)\" \"(%{HOSTPORT}|-)\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}", - # older - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" %{HOSTPORT} x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}" + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"" ] ] id => "cloudfoundry/app-rtr/grok" diff --git a/src/logsearch-config/src/logstash-filters/default.conf.erb b/src/logsearch-config/src/logstash-filters/default.conf.erb index 1beaf2dc..24a672ad 100644 --- a/src/logsearch-config/src/logstash-filters/default.conf.erb +++ b/src/logsearch-config/src/logstash-filters/default.conf.erb @@ -4,6 +4,6 @@ if [@type] in ["syslog", "relp"] { <%= File.read('src/logstash-filters/snippets/syslog_standard.conf').gsub(/^/, ' ') %> } -if [syslog_program] == "nats_to_syslog" { - <%= File.read('src/logstash-filters/snippets/bosh_nats.conf').gsub(/^/, ' ') %> -} +# if [syslog_program] == "nats_to_syslog" { +# <%= File.read('src/logstash-filters/snippets/bosh_nats.conf').gsub(/^/, ' ') %> +# } diff --git a/src/logsearch-filters/src/logstash-filters/default.conf.erb b/src/logsearch-filters/src/logstash-filters/default.conf.erb index c2bc8bd9..4928d990 100644 --- a/src/logsearch-filters/src/logstash-filters/default.conf.erb +++ b/src/logsearch-filters/src/logstash-filters/default.conf.erb @@ -22,12 +22,12 @@ ##-- Platform # (Platform snippet should precede all other platform snippets) -<%= File.read('src/logstash-filters/snippets/platform.conf') %> -# special cases parsing -<%= File.read('src/logstash-filters/snippets/platform-haproxy.conf') %> -<%= File.read('src/logstash-filters/snippets/platform-uaa.conf') %> -<%= File.read('src/logstash-filters/snippets/platform-vcap.conf') %> -<%= File.read('src/logstash-filters/snippets/platform-gorouter.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform.conf') %> +# # special cases parsing +# <%= File.read('src/logstash-filters/snippets/platform-haproxy.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform-uaa.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform-vcap.conf') %> +# <%= File.read('src/logstash-filters/snippets/platform-gorouter.conf') %> # Teardown snippet (should follow all other snippets) <%= File.read('src/logstash-filters/snippets/teardown.conf') %> diff --git a/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf index d74ba7da..810fc14d 100644 --- a/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf +++ b/src/logsearch-filters/src/logstash-filters/snippets/app-logmessage-rtr.conf @@ -11,15 +11,9 @@ if ( [@type] == "LogMessage" and [@source][type] == "RTR" ) { grok { match => [ "@message", [ # cf-deployment v12.27.0+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{TIMESTAMP_ISO8601})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"( %{GREEDYDATA:kvpairs})?", - # cf-deployment v12.17.0+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_time:%{NUMBER:[rtr][app_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"%{BASE10NUM:[rtr][app][index]:int|-}\"", + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{TIMESTAMP_ISO8601})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"( %{GREEDYDATA:kvpairs})?", # cf-release v252+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"", - # cf-release v250+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{HOSTPORT}|-)\" \"(%{HOSTPORT}|-)\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}", - # older - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" %{HOSTPORT} x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}" + "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"" ] ] id => "cloudfoundry/app-rtr/grok" From a524f22daef22dade5864c7f4cf9d898ee89548e Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Thu, 10 Oct 2024 15:37:31 -0400 Subject: [PATCH 2/2] removing unused comments and files --- .../src/logstash-filters/default.conf.erb | 4 - .../logstash-filters/snippets/bosh_nats.conf | 93 ------------------- .../src/logstash-filters/default.conf.erb | 9 -- .../platform-cloud_controller_ng.conf | 17 ---- .../snippets/platform-gorouter.conf | 35 ------- .../snippets/platform-haproxy.conf | 48 ---------- .../snippets/platform-uaa.conf | 71 -------------- .../snippets/platform-vcap.conf | 53 ----------- .../logstash-filters/snippets/platform.conf | 49 ---------- .../src/logstash-filters/default.conf.erb | 4 - .../logstash-filters/snippets/bosh_nats.conf | 93 ------------------- .../src/logstash-filters/default.conf.erb | 9 -- .../platform-cloud_controller_ng.conf | 17 ---- .../snippets/platform-gorouter.conf | 35 ------- .../snippets/platform-haproxy.conf | 48 ---------- .../snippets/platform-uaa.conf | 71 -------------- .../snippets/platform-vcap.conf | 53 ----------- .../logstash-filters/snippets/platform.conf | 49 ---------- 18 files changed, 758 deletions(-) delete mode 100644 src/base-logstash-filters/src/logstash-filters/snippets/bosh_nats.conf delete mode 100644 src/cf-logstash-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf delete mode 100644 src/cf-logstash-filters/src/logstash-filters/snippets/platform-gorouter.conf delete mode 100644 src/cf-logstash-filters/src/logstash-filters/snippets/platform-haproxy.conf delete mode 100644 src/cf-logstash-filters/src/logstash-filters/snippets/platform-uaa.conf delete mode 100644 src/cf-logstash-filters/src/logstash-filters/snippets/platform-vcap.conf delete mode 100644 src/cf-logstash-filters/src/logstash-filters/snippets/platform.conf delete mode 100644 src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf delete mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf delete mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf delete mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf delete mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf delete mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf delete mode 100644 src/logsearch-filters/src/logstash-filters/snippets/platform.conf diff --git a/src/base-logstash-filters/src/logstash-filters/default.conf.erb b/src/base-logstash-filters/src/logstash-filters/default.conf.erb index 24a672ad..7849a434 100644 --- a/src/base-logstash-filters/src/logstash-filters/default.conf.erb +++ b/src/base-logstash-filters/src/logstash-filters/default.conf.erb @@ -3,7 +3,3 @@ if [@type] in ["syslog", "relp"] { <%= File.read('src/logstash-filters/snippets/syslog_standard.conf').gsub(/^/, ' ') %> } - -# if [syslog_program] == "nats_to_syslog" { -# <%= File.read('src/logstash-filters/snippets/bosh_nats.conf').gsub(/^/, ' ') %> -# } diff --git a/src/base-logstash-filters/src/logstash-filters/snippets/bosh_nats.conf b/src/base-logstash-filters/src/logstash-filters/snippets/bosh_nats.conf deleted file mode 100644 index 3bacbcf5..00000000 --- a/src/base-logstash-filters/src/logstash-filters/snippets/bosh_nats.conf +++ /dev/null @@ -1,93 +0,0 @@ -# Parse BOSH NATS logs -if [syslog_program] == "nats_to_syslog" { - json { - source => "@message" - target => "NATS" - add_field => { "[@level]" => "INFO" } - add_tag => ["NATS"] - remove_field => ["@message"] - } - - if "_jsonparsefailure" in [tags] { - mutate { - add_tag => "fail/bosh_nats/json" - remove_tag => "_jsonparsefailure" - } - } else { - - json { - source => "[NATS][Data]" - target => "[NATS][Data]" - } - - if "_jsonparsefailure" in [tags] { - mutate { - add_tag => "fail/bosh_nats/Data/json" - remove_tag => "_jsonparsefailure" - } - } else { - if [NATS][Subject] =~ /hm\.agent\.heartbeat.*/ { - mutate { - add_field => { "[@source][vm]" => "%{[NATS][Data][job]}/%{[NATS][Data][index]}" } - add_tag => ["hm_agent_heartbeat"] - } - mutate { - rename => { "[NATS][Data][job]" => "[@source][job]" } - rename => { "[NATS][Data][index]" => "[@source][index]" } - } - - mutate { - convert => { "[NATS][Data][vitals][cpu][sys]" => "float" } - convert => { "[NATS][Data][vitals][cpu][user]" => "float" } - convert => { "[NATS][Data][vitals][cpu][wait]" => "float" } - convert => { "[NATS][Data][vitals][disk][ephemeral][inode_percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][ephemeral][percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][system][inode_percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][system][percent]" => "float" } - convert => { "[NATS][Data][vitals][mem][kb]" => "float" } - convert => { "[NATS][Data][vitals][mem][percent]" => "float" } - convert => { "[NATS][Data][vitals][swap][kb]" => "float" } - convert => { "[NATS][Data][vitals][swap][percent]" => "float" } - } - if [NATS][Data][vitals][disk][persistent] { - mutate { - convert => { "[NATS][Data][vitals][disk][persistent][inode_percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][persistent][percent]" => "float" } - } - } - ruby { - code => ' - vitals = event.get("NATS")["Data"]["vitals"].merge( {"load" => { - "avg01" => event.get("NATS")["Data"]["vitals"]["load"][0].to_f, - "avg05" => event.get("NATS")["Data"]["vitals"]["load"][1].to_f, - "avg15" => event.get("NATS")["Data"]["vitals"]["load"][2].to_f, - }}) - data = event.get("NATS")["Data"].merge({"vitals" => vitals}) - nats = event.get("NATS").merge({"Data" => data}) - event.set("NATS", nats) - ' - } - } else if [NATS][Subject] =~ /hm\.(director|agent)\.alert.*/ { - mutate { - add_tag => "hm_alert" - } - date { - match => [ "[NATS][Data][created_at]", "UNIX" ] - tag_on_failure => "fail/NATS/hm_alert/date" - remove_field => "[NATS][Data][created_at]" - } - translate { - source => "[NATS][Data][severity]" - target => "[@level]" - override => true - dictionary => [ - "1", "FATAL", - "2", "FATAL", - "3", "ERROR", - "4", "WARN" ] - fallback => "INFO" - } - } - } - } -} diff --git a/src/cf-logstash-filters/src/logstash-filters/default.conf.erb b/src/cf-logstash-filters/src/logstash-filters/default.conf.erb index e7232b6e..d3f834e3 100644 --- a/src/cf-logstash-filters/src/logstash-filters/default.conf.erb +++ b/src/cf-logstash-filters/src/logstash-filters/default.conf.erb @@ -17,14 +17,5 @@ <%= File.read('src/logstash-filters/snippets/app-metric.conf') %> <%= File.read('src/logstash-filters/snippets/app-http.conf') %> -##-- Platform -# (Platform snippet should precede all other platform snippets) -# <%= File.read('src/logstash-filters/snippets/platform.conf') %> -# # special cases parsing -# <%= File.read('src/logstash-filters/snippets/platform-haproxy.conf') %> -# <%= File.read('src/logstash-filters/snippets/platform-uaa.conf') %> -# <%= File.read('src/logstash-filters/snippets/platform-vcap.conf') %> -# <%= File.read('src/logstash-filters/snippets/platform-gorouter.conf') %> - # Teardown snippet (should follow all other snippets) <%= File.read('src/logstash-filters/snippets/teardown.conf') %> diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf deleted file mode 100644 index 19a19826..00000000 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf +++ /dev/null @@ -1,17 +0,0 @@ -##--------------------------------- -# Parses cloud_controller_ng logs.| -##--------------------------------- -if [@source][component] == "cloud_controller_ng" { - - mutate { - replace => { "@type" => "cloud_controller_ng" } - add_tag => "cloud_controller_ng" - } - - grok { - match => { "@message" => "%{URIHOST:Request_Host} %{NOTSPACE} \[%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{ISO8601_TIMEZONE}\] \"%{WORD:Request_Method} %{URIPATHPARAM:Request_URL} %{SYSLOGPROG:Request_Protocol}\" %{NUMBER:Status_Code:int} %{NUMBER:Bytes_Received:int} \"%{NOTSPACE:Referer}\" \"%{DATA:User_Agent}\" %{URIHOST:Backend_Address} vcap_request_id:%{DATA:X_Vcap_Request_ID} response_time:%{NUMBER:Response_Time}" } - tag_on_failure => "fail/cloudfoundry/platform-cloud_controller_ng/grok" - id => "cloudfoundry/platform-cloud_controller_ng/grok" - } - -} diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-gorouter.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/platform-gorouter.conf deleted file mode 100644 index 90065830..00000000 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-gorouter.conf +++ /dev/null @@ -1,35 +0,0 @@ -##------------------------------------+ -# Gorouter conf. Parses gorouter logs.| -##------------------------------------+ - -if [@index_type] == "platform" and [@source][component] == "gorouter" { - if [@message] =~ "\A\{.+\}\z" { - json { - source => "@message" - add_tag => [ "router/syslog" ] - tag_on_failure => [ "router/parsing_failed" ] - id => "router/accesslog/json" - } - } else { - grok { - match => [ "@message", [ - # cf-deployment v12.27.0+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{TIMESTAMP_ISO8601})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"( %{GREEDYDATA:kvpairs})?", - # cf-deployment v12.17.0+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_time:%{NUMBER:[rtr][app_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"%{BASE10NUM:[rtr][app][index]:int}\"", - # cf-release v252+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"", - # cf-release v250+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{HOSTPORT}|-)\" \"(%{HOSTPORT}|-)\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}", - # very old - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" %{HOSTPORT} x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}" - ] - ] - id => "router/accesslog/grok" - overwrite => [ "@message" ] - add_tag => "router/accesslog" - tag_on_failure => "router/parsing_failed" - } - - } -} diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-haproxy.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/platform-haproxy.conf deleted file mode 100644 index b130d041..00000000 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-haproxy.conf +++ /dev/null @@ -1,48 +0,0 @@ -##---------------------------------- -# Haproxy conf. Parses haproxy logs.| -##---------------------------------- -if [@source][component] == "haproxy" { - - mutate { - replace => { "@type" => "haproxy" } - add_tag => "haproxy" - } - - # Grok patterns are based on http://www.haproxy.org/download/1.7/doc/configuration.txt - # Two formats are used accordingly: - # 8.2.3. HTTP log format - # 8.2.5. Error log format - - grok { - match => [ "@message", "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][server_name]} %{INT:[haproxy][time_request]:int}/%{INT:[haproxy][time_queue]:int}/%{INT:[haproxy][time_backend_connect]:int}/%{INT:[haproxy][time_backend_response]:int}/%{INT:[haproxy][time_duration]:int} %{INT:[haproxy][http_status_code]:int} %{NOTSPACE:[haproxy][bytes_read]:int} %{DATA:[haproxy][captured_request_cookie]} %{DATA:[haproxy][captured_response_cookie]} %{NOTSPACE:[haproxy][termination_state]} %{INT:[haproxy][actconn]:int}/%{INT:[haproxy][feconn]:int}/%{INT:[haproxy][beconn]:int}/%{INT:[haproxy][srvconn]:int}/%{NOTSPACE:[haproxy][retries]:int} %{INT:[haproxy][srv_queue]:int}/%{INT:[haproxy][backend_queue]:int} (\{%{DATA:[haproxy][captured_request_headers]}\})?( )?(\{%{DATA:[haproxy][captured_response_headers]}\})?( )?\"(?(?(|((%{WORD:[haproxy][http_request_verb]})?( %{GREEDYDATA})?))))\"" ] - match => [ "@message", "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]}/%{NOTSPACE:[haproxy][bind_name]}:%{SPACE}%{GREEDYDATA:message}" ] - id => "cloudfoundry/platform-haproxy/grok" - tag_on_failure => "fail/cloudfoundry/platform-haproxy/grok" - } - - if !("fail/cloudfoundry/platform-haproxy/grok" in [tags]) { - - if [haproxy_http_request] { - mutate { - rename => {"haproxy_http_request" => "[haproxy][http_request]"} - } - } - - mutate { - rename => {"message" => "@message"} # @message - } - - # @level - if [haproxy][http_status_code] { - if [haproxy][http_status_code] >= 400 { - mutate { - add_field => { "@level" => "ERROR" } - } - } else { - mutate { - add_field => { "@level" => "INFO" } - } - } - } - } -} diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-uaa.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/platform-uaa.conf deleted file mode 100644 index eb3d52f2..00000000 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-uaa.conf +++ /dev/null @@ -1,71 +0,0 @@ -##-------------------------- -# Uaa conf. Parses uaa logs.| -##-------------------------- -if [@source][component] == "vcap.uaa" { - - # ---- Parse UAA events (general) - - mutate { - replace => { "[@source][component]" => "uaa" } # remove vcap. prefix - replace => { "@type" => "uaa" } - add_tag => "uaa" - } - - grok { - match => { "@message" => "\[%{TIMESTAMP_ISO8601:[uaa][timestamp]}\]%{SPACE}uaa%{SPACE}-%{SPACE}%{NUMBER:[uaa][pid]:int}%{SPACE}\[%{DATA:[uaa][thread]}\]%{SPACE}....%{SPACE}%{LOGLEVEL:@level}%{SPACE}---%{SPACE}%{DATA:[uaa][log_category]}:%{SPACE}%{GREEDYDATA:@message}"} - overwrite => ["@message", "@level"] # @message, @level - id => "cloudfoundry/platform-uaa/grok" - tag_on_failure => "fail/cloudfoundry/platform-uaa/grok" - } - - if [uaa][log_category] == "Audit" { - - # override - mutate { - replace => { "@type" => "uaa-audit" } - add_tag => "audit" - } - - # ---- Additional parsing: Audit events - - grok { - match => { "@message" => "(?(%{WORD:[uaa][audit][type]}%{SPACE}\('%{DATA:[uaa][audit][data]}'\))):%{SPACE}principal=%{DATA:[uaa][audit][principal]},%{SPACE}origin=\[%{DATA:[uaa][audit][origin]}\],%{SPACE}identityZoneId=\[%{DATA:[uaa][audit][identity_zone_id]}\]"} - id => "cloudfoundry/platform-uaa/audit/grok" - tag_on_failure => "fail/cloudfoundry/platform-uaa/audit/grok" - } - - if !("fail/cloudfoundry/platform-uaa/audit/grok" in [tags]) { - - # Audit @message - mutate { - rename => { "uaa_audit_message" => "@message" } - } - - # extract audit_event_remote_address and geoip it - if "PrincipalAuthenticationFailure" == [uaa][audit][type] { - mutate { - add_field => { "[uaa][audit][remote_address]" => "%{[uaa][audit][origin]}" } - } - } - if [uaa][audit][origin] =~ /remoteAddress=/ { - grok { - match => { "[uaa][audit][origin]" => "remoteAddress=%{IP:[uaa][audit][remote_address]}" } - id => "cloudfoundry/platform-uaa/audit/origin/grok" - } - } - if [uaa][audit][remote_address] { - geoip { - source => "[uaa][audit][remote_address]" - } - } - - # split origin - mutate { - split => { "[uaa][audit][origin]" => ", " } - } - - } - - } -} - diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-vcap.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/platform-vcap.conf deleted file mode 100644 index b952e34b..00000000 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/platform-vcap.conf +++ /dev/null @@ -1,53 +0,0 @@ -##----------------------------- -# Vcap conf. Parses vcap* logs.| -##----------------------------- -if [@source][component] != "vcap.uaa" and [@source][component] =~ /vcap\..*/ { - - # minus vcap. prefix - mutate { - gsub => ["[@source][component]", "^vcap\.", ""] - } - - mutate { - replace => { "@type" => "vcap" } - add_tag => "vcap" - } - - # Parse Cloud Foundry logs - if [@message] =~ /^\s*{".*}\s*$/ { # looks like JSON - - # parse JSON message - json { - source => "@message" - target => "parsed_json_field" - remove_field => [ "@message" ] - add_field => { "parsed_json_field_name" => "%{[@source][component]}"} - id => "cloudfoundry/platform-vcap/json" - } - - if "_jsonparsefailure" in [tags] { - # Amend the failure tag to match our fail/${addon}/${filter}/${detail} standard - mutate { - add_tag => ["fail/cloudfoundry/platform-vcap/json"] - remove_tag => ["_jsonparsefailure"] - } - - } else { - - mutate { - rename => { "[parsed_json_field][message]" => "@message" } # @message - } - - # @level - translate { - source => "[parsed_json_field][log_level]" - dictionary => [ "0", "DEBUG", "1", "INFO", "2", "ERROR", "3", "FATAL" ] - target => "@level" - override => true - fallback => "%{[parsed_json_field][log_level]}" - remove_field => "[parsed_json_field][log_level]" - } - } - - } -} diff --git a/src/cf-logstash-filters/src/logstash-filters/snippets/platform.conf b/src/cf-logstash-filters/src/logstash-filters/snippets/platform.conf deleted file mode 100644 index 787c84aa..00000000 --- a/src/cf-logstash-filters/src/logstash-filters/snippets/platform.conf +++ /dev/null @@ -1,49 +0,0 @@ -##------------------------------ -# Platform conf. Parses CF logs.| -##------------------------------ -if [@index_type] == "platform" { - - mutate { - replace => { "[@source][type]" => "system" } # default for platform logs - add_tag => "platform" - } - - # Syslog message with RFC 5424 and the enterprise number is CF - if [syslog_sd_id] == "instance@47450" { - mutate { - add_field => { - "[@source][az]" => "%{[syslog_sd_params][az]}" - "[@source][deployment]" => "%{[syslog_sd_params][deployment]}" - "[@source][director]" => "%{[syslog_sd_params][director]}" - "[@source][id]" => "%{[syslog_sd_params][id]}" - "[@source][job]" => "%{[syslog_sd_params][group]}" - } - replace => { - "[@source][type]" => "cf" - "@type" => "cf" - } - add_tag => "cf" - } - } else { - # Try parsing with possible CF formats - grok { - # Metron agent format (https://github.com/cloudfoundry/loggregator/blob/master/jobs/metron_agent/templates/syslog_forwarder.conf.erb#L53) - match => [ "@message", "\[job=%{NOTSPACE:[@source][job]} index=%{INT:[@source][index]:int}\]%{SPACE}%{GREEDYDATA:@message}" ] - - # Syslog release format (https://github.com/cloudfoundry/syslog-release/blob/master/jobs/syslog_forwarder/templates/rsyslog.conf.erb#L56) - match => [ "@message", "\[bosh instance=%{NOTSPACE:[@source][deployment]}/%{NOTSPACE:[@source][job]}/%{NOTSPACE:[@source][job_index]}\]%{SPACE}%{GREEDYDATA:@message}" ] - - overwrite => [ "@message" ] # @message - id => "cloudfoundry/platform/grok" - tag_on_failure => [ "app_source"] - } - - if !("fail/cloudfoundry/platform/grok" in [tags]) { - mutate { - replace => { "[@source][type]" => "cf" } - replace => { "@type" => "cf" } - add_tag => "cf" - } - } - } -} diff --git a/src/logsearch-config/src/logstash-filters/default.conf.erb b/src/logsearch-config/src/logstash-filters/default.conf.erb index 24a672ad..7849a434 100644 --- a/src/logsearch-config/src/logstash-filters/default.conf.erb +++ b/src/logsearch-config/src/logstash-filters/default.conf.erb @@ -3,7 +3,3 @@ if [@type] in ["syslog", "relp"] { <%= File.read('src/logstash-filters/snippets/syslog_standard.conf').gsub(/^/, ' ') %> } - -# if [syslog_program] == "nats_to_syslog" { -# <%= File.read('src/logstash-filters/snippets/bosh_nats.conf').gsub(/^/, ' ') %> -# } diff --git a/src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf b/src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf deleted file mode 100644 index 28fc1e31..00000000 --- a/src/logsearch-config/src/logstash-filters/snippets/bosh_nats.conf +++ /dev/null @@ -1,93 +0,0 @@ -# Parse BOSH NATS logs -if [syslog_program] == "nats_to_syslog" { - json { - source => "@message" - target => "NATS" - add_field => { "[@level]" => "INFO" } - add_tag => ["NATS"] - remove_field => ["@message"] - } - - if "_jsonparsefailure" in [tags] { - mutate { - add_tag => "fail/bosh_nats/json" - remove_tag => "_jsonparsefailure" - } - } else { - - json { - source => "[NATS][Data]" - target => "[NATS][Data]" - } - - if "_jsonparsefailure" in [tags] { - mutate { - add_tag => "fail/bosh_nats/Data/json" - remove_tag => "_jsonparsefailure" - } - } else { - if [NATS][Subject] =~ /hm\.agent\.heartbeat.*/ { - mutate { - add_field => { "[@source][vm]" => "%{[NATS][Data][job]}/%{[NATS][Data][index]}" } - add_tag => ["hm_agent_heartbeat"] - } - mutate { - rename => { "[NATS][Data][job]" => "[@source][job]" } - rename => { "[NATS][Data][index]" => "[@source][index]" } - } - - mutate { - convert => { "[NATS][Data][vitals][cpu][sys]" => "float" } - convert => { "[NATS][Data][vitals][cpu][user]" => "float" } - convert => { "[NATS][Data][vitals][cpu][wait]" => "float" } - convert => { "[NATS][Data][vitals][disk][ephemeral][inode_percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][ephemeral][percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][system][inode_percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][system][percent]" => "float" } - convert => { "[NATS][Data][vitals][mem][kb]" => "float" } - convert => { "[NATS][Data][vitals][mem][percent]" => "float" } - convert => { "[NATS][Data][vitals][swap][kb]" => "float" } - convert => { "[NATS][Data][vitals][swap][percent]" => "float" } - } - if [NATS][Data][vitals][disk][persistent] { - mutate { - convert => { "[NATS][Data][vitals][disk][persistent][inode_percent]" => "float" } - convert => { "[NATS][Data][vitals][disk][persistent][percent]" => "float" } - } - } - ruby { - code => ' - vitals = event.get("NATS")["Data"]["vitals"].merge( {"load" => { - "avg01" => event.get("NATS")["Data"]["vitals"]["load"][0].to_f, - "avg05" => event.get("NATS")["Data"]["vitals"]["load"][1].to_f, - "avg15" => event.get("NATS")["Data"]["vitals"]["load"][2].to_f, - }}) - data = event.get("NATS")["Data"].merge({"vitals" => vitals}) - nats = event.get("NATS").merge({"Data" => data}) - event.set("NATS", nats) - ' - } - } else if [NATS][Subject] =~ /hm\.(director|agent)\.alert.*/ { - mutate { - add_tag => "hm_alert" - } - date { - match => [ "[NATS][Data][created_at]", "UNIX" ] - tag_on_failure => "fail/NATS/hm_alert/date" - remove_field => "[NATS][Data][created_at]" - } - translate { - field => "[NATS][Data][severity]" - destination => "[@level]" - override => true - dictionary => [ - "1", "FATAL", - "2", "FATAL", - "3", "ERROR", - "4", "WARN" ] - fallback => "INFO" - } - } - } - } -} diff --git a/src/logsearch-filters/src/logstash-filters/default.conf.erb b/src/logsearch-filters/src/logstash-filters/default.conf.erb index 4928d990..4b084fd4 100644 --- a/src/logsearch-filters/src/logstash-filters/default.conf.erb +++ b/src/logsearch-filters/src/logstash-filters/default.conf.erb @@ -20,14 +20,5 @@ <%= File.read('src/logstash-filters/snippets/app-counterevent.conf') %> <%= File.read('src/logstash-filters/snippets/app-http.conf') %> -##-- Platform -# (Platform snippet should precede all other platform snippets) -# <%= File.read('src/logstash-filters/snippets/platform.conf') %> -# # special cases parsing -# <%= File.read('src/logstash-filters/snippets/platform-haproxy.conf') %> -# <%= File.read('src/logstash-filters/snippets/platform-uaa.conf') %> -# <%= File.read('src/logstash-filters/snippets/platform-vcap.conf') %> -# <%= File.read('src/logstash-filters/snippets/platform-gorouter.conf') %> - # Teardown snippet (should follow all other snippets) <%= File.read('src/logstash-filters/snippets/teardown.conf') %> diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf deleted file mode 100644 index 19a19826..00000000 --- a/src/logsearch-filters/src/logstash-filters/snippets/platform-cloud_controller_ng.conf +++ /dev/null @@ -1,17 +0,0 @@ -##--------------------------------- -# Parses cloud_controller_ng logs.| -##--------------------------------- -if [@source][component] == "cloud_controller_ng" { - - mutate { - replace => { "@type" => "cloud_controller_ng" } - add_tag => "cloud_controller_ng" - } - - grok { - match => { "@message" => "%{URIHOST:Request_Host} %{NOTSPACE} \[%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{ISO8601_TIMEZONE}\] \"%{WORD:Request_Method} %{URIPATHPARAM:Request_URL} %{SYSLOGPROG:Request_Protocol}\" %{NUMBER:Status_Code:int} %{NUMBER:Bytes_Received:int} \"%{NOTSPACE:Referer}\" \"%{DATA:User_Agent}\" %{URIHOST:Backend_Address} vcap_request_id:%{DATA:X_Vcap_Request_ID} response_time:%{NUMBER:Response_Time}" } - tag_on_failure => "fail/cloudfoundry/platform-cloud_controller_ng/grok" - id => "cloudfoundry/platform-cloud_controller_ng/grok" - } - -} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf deleted file mode 100644 index 90065830..00000000 --- a/src/logsearch-filters/src/logstash-filters/snippets/platform-gorouter.conf +++ /dev/null @@ -1,35 +0,0 @@ -##------------------------------------+ -# Gorouter conf. Parses gorouter logs.| -##------------------------------------+ - -if [@index_type] == "platform" and [@source][component] == "gorouter" { - if [@message] =~ "\A\{.+\}\z" { - json { - source => "@message" - add_tag => [ "router/syslog" ] - tag_on_failure => [ "router/parsing_failed" ] - id => "router/accesslog/json" - } - } else { - grok { - match => [ "@message", [ - # cf-deployment v12.27.0+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{TIMESTAMP_ISO8601})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"( %{GREEDYDATA:kvpairs})?", - # cf-deployment v12.17.0+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} gorouter_time:%{NUMBER:[rtr][gorouter_time_sec]:float} app_time:%{NUMBER:[rtr][app_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"%{BASE10NUM:[rtr][app][index]:int}\"", - # cf-release v252+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}|-)\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"(%{BASE10NUM:[rtr][app][index]:int}|-)\"", - # cf-release v250+ - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"(%{HOSTPORT}|-)\" \"(%{HOSTPORT}|-)\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}", - # very old - "^%{HOSTNAME:[rtr][hostname]} - \[(?%{MONTHDAY}/%{MONTHNUM}/%{YEAR}:%{TIME} %{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" %{HOSTPORT} x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:%{NOTSPACE}%{GREEDYDATA}" - ] - ] - id => "router/accesslog/grok" - overwrite => [ "@message" ] - add_tag => "router/accesslog" - tag_on_failure => "router/parsing_failed" - } - - } -} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf deleted file mode 100644 index b130d041..00000000 --- a/src/logsearch-filters/src/logstash-filters/snippets/platform-haproxy.conf +++ /dev/null @@ -1,48 +0,0 @@ -##---------------------------------- -# Haproxy conf. Parses haproxy logs.| -##---------------------------------- -if [@source][component] == "haproxy" { - - mutate { - replace => { "@type" => "haproxy" } - add_tag => "haproxy" - } - - # Grok patterns are based on http://www.haproxy.org/download/1.7/doc/configuration.txt - # Two formats are used accordingly: - # 8.2.3. HTTP log format - # 8.2.5. Error log format - - grok { - match => [ "@message", "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][server_name]} %{INT:[haproxy][time_request]:int}/%{INT:[haproxy][time_queue]:int}/%{INT:[haproxy][time_backend_connect]:int}/%{INT:[haproxy][time_backend_response]:int}/%{INT:[haproxy][time_duration]:int} %{INT:[haproxy][http_status_code]:int} %{NOTSPACE:[haproxy][bytes_read]:int} %{DATA:[haproxy][captured_request_cookie]} %{DATA:[haproxy][captured_response_cookie]} %{NOTSPACE:[haproxy][termination_state]} %{INT:[haproxy][actconn]:int}/%{INT:[haproxy][feconn]:int}/%{INT:[haproxy][beconn]:int}/%{INT:[haproxy][srvconn]:int}/%{NOTSPACE:[haproxy][retries]:int} %{INT:[haproxy][srv_queue]:int}/%{INT:[haproxy][backend_queue]:int} (\{%{DATA:[haproxy][captured_request_headers]}\})?( )?(\{%{DATA:[haproxy][captured_response_headers]}\})?( )?\"(?(?(|((%{WORD:[haproxy][http_request_verb]})?( %{GREEDYDATA})?))))\"" ] - match => [ "@message", "%{IP:[haproxy][client_ip]}:%{INT:[haproxy][client_port]:int} \[%{DATA:[haproxy][accept_date]}\] %{NOTSPACE:[haproxy][frontend_name]}/%{NOTSPACE:[haproxy][bind_name]}:%{SPACE}%{GREEDYDATA:message}" ] - id => "cloudfoundry/platform-haproxy/grok" - tag_on_failure => "fail/cloudfoundry/platform-haproxy/grok" - } - - if !("fail/cloudfoundry/platform-haproxy/grok" in [tags]) { - - if [haproxy_http_request] { - mutate { - rename => {"haproxy_http_request" => "[haproxy][http_request]"} - } - } - - mutate { - rename => {"message" => "@message"} # @message - } - - # @level - if [haproxy][http_status_code] { - if [haproxy][http_status_code] >= 400 { - mutate { - add_field => { "@level" => "ERROR" } - } - } else { - mutate { - add_field => { "@level" => "INFO" } - } - } - } - } -} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf deleted file mode 100644 index eb3d52f2..00000000 --- a/src/logsearch-filters/src/logstash-filters/snippets/platform-uaa.conf +++ /dev/null @@ -1,71 +0,0 @@ -##-------------------------- -# Uaa conf. Parses uaa logs.| -##-------------------------- -if [@source][component] == "vcap.uaa" { - - # ---- Parse UAA events (general) - - mutate { - replace => { "[@source][component]" => "uaa" } # remove vcap. prefix - replace => { "@type" => "uaa" } - add_tag => "uaa" - } - - grok { - match => { "@message" => "\[%{TIMESTAMP_ISO8601:[uaa][timestamp]}\]%{SPACE}uaa%{SPACE}-%{SPACE}%{NUMBER:[uaa][pid]:int}%{SPACE}\[%{DATA:[uaa][thread]}\]%{SPACE}....%{SPACE}%{LOGLEVEL:@level}%{SPACE}---%{SPACE}%{DATA:[uaa][log_category]}:%{SPACE}%{GREEDYDATA:@message}"} - overwrite => ["@message", "@level"] # @message, @level - id => "cloudfoundry/platform-uaa/grok" - tag_on_failure => "fail/cloudfoundry/platform-uaa/grok" - } - - if [uaa][log_category] == "Audit" { - - # override - mutate { - replace => { "@type" => "uaa-audit" } - add_tag => "audit" - } - - # ---- Additional parsing: Audit events - - grok { - match => { "@message" => "(?(%{WORD:[uaa][audit][type]}%{SPACE}\('%{DATA:[uaa][audit][data]}'\))):%{SPACE}principal=%{DATA:[uaa][audit][principal]},%{SPACE}origin=\[%{DATA:[uaa][audit][origin]}\],%{SPACE}identityZoneId=\[%{DATA:[uaa][audit][identity_zone_id]}\]"} - id => "cloudfoundry/platform-uaa/audit/grok" - tag_on_failure => "fail/cloudfoundry/platform-uaa/audit/grok" - } - - if !("fail/cloudfoundry/platform-uaa/audit/grok" in [tags]) { - - # Audit @message - mutate { - rename => { "uaa_audit_message" => "@message" } - } - - # extract audit_event_remote_address and geoip it - if "PrincipalAuthenticationFailure" == [uaa][audit][type] { - mutate { - add_field => { "[uaa][audit][remote_address]" => "%{[uaa][audit][origin]}" } - } - } - if [uaa][audit][origin] =~ /remoteAddress=/ { - grok { - match => { "[uaa][audit][origin]" => "remoteAddress=%{IP:[uaa][audit][remote_address]}" } - id => "cloudfoundry/platform-uaa/audit/origin/grok" - } - } - if [uaa][audit][remote_address] { - geoip { - source => "[uaa][audit][remote_address]" - } - } - - # split origin - mutate { - split => { "[uaa][audit][origin]" => ", " } - } - - } - - } -} - diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf deleted file mode 100644 index 768dcb56..00000000 --- a/src/logsearch-filters/src/logstash-filters/snippets/platform-vcap.conf +++ /dev/null @@ -1,53 +0,0 @@ -##----------------------------- -# Vcap conf. Parses vcap* logs.| -##----------------------------- -if [@source][component] != "vcap.uaa" and [@source][component] =~ /vcap\..*/ { - - # minus vcap. prefix - mutate { - gsub => ["[@source][component]", "^vcap\.", ""] - } - - mutate { - replace => { "@type" => "vcap" } - add_tag => "vcap" - } - - # Parse Cloud Foundry logs - if [@message] =~ /^\s*{".*}\s*$/ { # looks like JSON - - # parse JSON message - json { - source => "@message" - target => "parsed_json_field" - remove_field => [ "@message" ] - add_field => { "parsed_json_field_name" => "%{[@source][component]}"} - id => "cloudfoundry/platform-vcap/json" - } - - if "_jsonparsefailure" in [tags] { - # Amend the failure tag to match our fail/${addon}/${filter}/${detail} standard - mutate { - add_tag => ["fail/cloudfoundry/platform-vcap/json"] - remove_tag => ["_jsonparsefailure"] - } - - } else { - - mutate { - rename => { "[parsed_json_field][message]" => "@message" } # @message - } - - # @level - translate { - field => "[parsed_json_field][log_level]" - dictionary => [ "0", "DEBUG", "1", "INFO", "2", "ERROR", "3", "FATAL" ] - destination => "@level" - override => true - fallback => "%{[parsed_json_field][log_level]}" - remove_field => "[parsed_json_field][log_level]" - } - } - - } -} diff --git a/src/logsearch-filters/src/logstash-filters/snippets/platform.conf b/src/logsearch-filters/src/logstash-filters/snippets/platform.conf deleted file mode 100644 index c5c8d5a7..00000000 --- a/src/logsearch-filters/src/logstash-filters/snippets/platform.conf +++ /dev/null @@ -1,49 +0,0 @@ -##------------------------------ -# Platform conf. Parses CF logs.| -##------------------------------ -if [@index_type] == "platform" { - - mutate { - replace => { "[@source][type]" => "system" } # default for platform logs - add_tag => "platform" - } - - # Syslog message with RFC 5424 and the enterprise number is CF - if [syslog_sd_id] == "instance@47450" { - mutate { - add_field => { - "[@source][az]" => "%{[syslog_sd_params][az]}" - "[@source][deployment]" => "%{[syslog_sd_params][deployment]}" - "[@source][director]" => "%{[syslog_sd_params][director]}" - "[@source][id]" => "%{[syslog_sd_params][id]}" - "[@source][job]" => "%{[syslog_sd_params][group]}" - } - replace => { - "[@source][type]" => "cf" - "@type" => "cf" - } - add_tag => "cf" - } - } else { - # Try parsing with possible CF formats - grok { - # Metron agent format (https://github.com/cloudfoundry/loggregator/blob/master/jobs/metron_agent/templates/syslog_forwarder.conf.erb#L53) - match => [ "@message", "\[job=%{NOTSPACE:[@source][job]} index=%{INT:[@source][index]:int}\]%{SPACE}%{GREEDYDATA:@message}" ] - - # Syslog release format (https://github.com/cloudfoundry/syslog-release/blob/master/jobs/syslog_forwarder/templates/rsyslog.conf.erb#L56) - match => [ "@message", "\[bosh instance=%{NOTSPACE:[@source][deployment]}/%{NOTSPACE:[@source][job]}/%{NOTSPACE:[@source][job_index]}\]%{SPACE}%{GREEDYDATA:@message}" ] - - overwrite => [ "@message" ] # @message - id => "cloudfoundry/platform/grok" - tag_on_failure => "fail/cloudfoundry/platform/grok" - } - - if !("fail/cloudfoundry/platform/grok" in [tags]) { - mutate { - replace => { "[@source][type]" => "cf" } - replace => { "@type" => "cf" } - add_tag => "cf" - } - } - } -}