From 71dae522d6ddc62487fbf582cf965d0cb97bcf74 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 18 Nov 2024 16:26:44 -0500 Subject: [PATCH 01/17] attempt at fixing filter ability --- jobs/ingestor_syslog/templates/bin/ingestor_syslog | 4 ++++ jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/jobs/ingestor_syslog/templates/bin/ingestor_syslog b/jobs/ingestor_syslog/templates/bin/ingestor_syslog index 4d183df0..92f474af 100644 --- a/jobs/ingestor_syslog/templates/bin/ingestor_syslog +++ b/jobs/ingestor_syslog/templates/bin/ingestor_syslog @@ -91,7 +91,11 @@ cat ${JOB_DIR}/config/filters_post.conf >> ${JOB_DIR}/config/logstash.conf <% if p('logstash_parser.enable_json_filter') %> cat /var/vcap/packages/base-logstash-filters/if_it_looks_like_json.conf >> ${JOB_DIR}/config/logstash.conf <% end %> + +<% if p('logstash_ingestor.cf-enabled') %> cat /var/vcap/packages/base-logstash-filters/timecop.conf >> ${JOB_DIR}/config/logstash.conf +<% end %> + cat /var/vcap/packages/base-logstash-filters/deployment.conf >> ${JOB_DIR}/config/logstash.conf echo "} #close filters" >> ${JOB_DIR}/config/logstash.conf diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index eead03e7..7f686991 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -25,7 +25,12 @@ # # Injected custom filters below from logstash_inject.filters manifest property # - <%= p('logstash_ingestor.filters') %> + <% p('logstash_ingestor.filters').each do | filter | %> + <%= filter['pattern'] %> { + + <% filter['options'].each do | k, v | %> + <%= k %> => <%= v.inspect %> + <% end %> <% if 'DEBUG' == p('logstash.metadata_level') %> From 77d59c2b8c663a8185353cf21a748621e5615465 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 18 Nov 2024 16:47:56 -0500 Subject: [PATCH 02/17] second end --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 7f686991..b97b34b5 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -31,7 +31,7 @@ <% filter['options'].each do | k, v | %> <%= k %> => <%= v.inspect %> <% end %> - + <% end %> <% if 'DEBUG' == p('logstash.metadata_level') %> mutate { From b6090fecc86bc02ce9f65e84dc2b1668fa0dbcdc Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 18 Nov 2024 16:49:47 -0500 Subject: [PATCH 03/17] second end --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index b97b34b5..82383616 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -31,6 +31,7 @@ <% filter['options'].each do | k, v | %> <%= k %> => <%= v.inspect %> <% end %> + } <% end %> <% if 'DEBUG' == p('logstash.metadata_level') %> From 32f9db98182d6db1b6908282b9100e801083eb40 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Mon, 18 Nov 2024 16:52:40 -0500 Subject: [PATCH 04/17] second end --- .../templates/config/filters_pre.conf.erb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 82383616..27cb5f55 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -25,14 +25,13 @@ # # Injected custom filters below from logstash_inject.filters manifest property # - <% p('logstash_ingestor.filters').each do | filter | %> - <%= filter['pattern'] %> { - - <% filter['options'].each do | k, v | %> - <%= k %> => <%= v.inspect %> - <% end %> - } - <% end %> + <% p("logstash_ingestor.filters", []).each do | filter | %> + <%= filter["pattern"] %> { + <% filter["options"].each do | k, v | %> + <%= k %> => <%= v.inspect %> + <% end %> + } + <% end %> <% if 'DEBUG' == p('logstash.metadata_level') %> mutate { From 6ac511790b7d8a3da25d046b55a6a94d0a831ccf Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 09:23:54 -0500 Subject: [PATCH 05/17] adding in different formatting --- .../ingestor_syslog/templates/config/filters_pre.conf.erb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 27cb5f55..1b465b04 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -25,13 +25,15 @@ # # Injected custom filters below from logstash_inject.filters manifest property # + <% p("logstash_ingestor.filters", []).each do | filter | %> - <%= filter["pattern"] %> { + <%= filter["pattern"] %> + { <% filter["options"].each do | k, v | %> <%= k %> => <%= v.inspect %> <% end %> - } - <% end %> + } + <% end %> <% if 'DEBUG' == p('logstash.metadata_level') %> mutate { From 57e50955c886858edc668d64f2e37d36792ec465 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 10:07:50 -0500 Subject: [PATCH 06/17] adding in s3 access --- .../ingestor_syslog/templates/config/filters_pre.conf.erb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 1b465b04..bdb48cbf 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -33,6 +33,14 @@ <%= k %> => <%= v.inspect %> <% end %> } + + <% if p('logstash_ingestor.cf-enabled') %> + grok { + match: { "@message" => "%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{GREEDYDATA:@message}$" } +442 tag_on_failure: ["fail/s3-start/_grokparsefailure"] + } + <% end %> + <% end %> <% if 'DEBUG' == p('logstash.metadata_level') %> From 6ca687300ebf41662f8dcf69e3b644a28dc5e561 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 10:09:29 -0500 Subject: [PATCH 07/17] can't be none --- .../templates/config/filters_pre.conf.erb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index bdb48cbf..cf047309 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -26,13 +26,13 @@ # Injected custom filters below from logstash_inject.filters manifest property # - <% p("logstash_ingestor.filters", []).each do | filter | %> - <%= filter["pattern"] %> - { - <% filter["options"].each do | k, v | %> - <%= k %> => <%= v.inspect %> - <% end %> - } + # <% p("logstash_ingestor.filters", []).each do | filter | %> + # <%= filter["pattern"] %> + # { + # <% filter["options"].each do | k, v | %> + # <%= k %> => <%= v.inspect %> + # <% end %> + # } <% if p('logstash_ingestor.cf-enabled') %> grok { From 287b7963ac05b3c00fb5cfd0d0adf0620b70da70 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 10:11:50 -0500 Subject: [PATCH 08/17] can't be none --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index cf047309..b2d9bffb 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -33,6 +33,7 @@ # <%= k %> => <%= v.inspect %> # <% end %> # } + # <% end %> <% if p('logstash_ingestor.cf-enabled') %> grok { @@ -41,8 +42,6 @@ } <% end %> - <% end %> - <% if 'DEBUG' == p('logstash.metadata_level') %> mutate { add_field => [ "@parser[job]", "<%= name %>/<%= index %>" ] From 6cd3533b134c77e0f0a551c403b6c69695be2f65 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 10:14:37 -0500 Subject: [PATCH 09/17] removing because it errors --- .../templates/config/filters_pre.conf.erb | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index b2d9bffb..ec1e5c8a 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -26,19 +26,11 @@ # Injected custom filters below from logstash_inject.filters manifest property # - # <% p("logstash_ingestor.filters", []).each do | filter | %> - # <%= filter["pattern"] %> - # { - # <% filter["options"].each do | k, v | %> - # <%= k %> => <%= v.inspect %> - # <% end %> - # } - # <% end %> <% if p('logstash_ingestor.cf-enabled') %> grok { match: { "@message" => "%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{GREEDYDATA:@message}$" } -442 tag_on_failure: ["fail/s3-start/_grokparsefailure"] + tag_on_failure: ["fail/s3-start/_grokparsefailure"] } <% end %> From 3a2b29999d896d67f720234b9af9a3d588a0c7ab Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 10:23:36 -0500 Subject: [PATCH 10/17] removing because it errors --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index ec1e5c8a..6ffde45e 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -29,8 +29,8 @@ <% if p('logstash_ingestor.cf-enabled') %> grok { - match: { "@message" => "%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{GREEDYDATA:@message}$" } - tag_on_failure: ["fail/s3-start/_grokparsefailure"] + match => { "@message" => "^%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{GREEDYDATA:@message}$" } + tag_on_failure => ["fail/s3-start/_grokparsefailure"] } <% end %> From ba6ad1ddf619c7fd0e1a737ba501354db0cd49ff Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 10:31:32 -0500 Subject: [PATCH 11/17] fixing cf_enabled --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 6ffde45e..078561df 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -27,7 +27,7 @@ # - <% if p('logstash_ingestor.cf-enabled') %> + <% if !p('logstash_ingestor.cf-enabled') %> grok { match => { "@message" => "^%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{GREEDYDATA:@message}$" } tag_on_failure => ["fail/s3-start/_grokparsefailure"] From 08536d9e292f02cae0b9cb4b10675d017df7e8b2 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 11:02:05 -0500 Subject: [PATCH 12/17] fixing cf_enabled --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 078561df..9853dc75 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -29,7 +29,7 @@ <% if !p('logstash_ingestor.cf-enabled') %> grok { - match => { "@message" => "^%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{GREEDYDATA:@message}$" } + match => { "message" => "^%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{SPACE}%{GREEDYDATA:message}$" } tag_on_failure => ["fail/s3-start/_grokparsefailure"] } <% end %> From 89f8f9a82bd2206ca258a7cd706cc258ce5fa52f Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 11:11:43 -0500 Subject: [PATCH 13/17] trying to remove old values --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 9853dc75..a21791ec 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -29,7 +29,7 @@ <% if !p('logstash_ingestor.cf-enabled') %> grok { - match => { "message" => "^%{TIMESTAMP_ISO8601}%{SPACE}%{SYSLOGPROG}%{SPACE}%{GREEDYDATA:message}$" } + match => { "message" => "^%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{SYSLOGPROG:program}%{SPACE}%{GREEDYDATA:message}$" } tag_on_failure => ["fail/s3-start/_grokparsefailure"] } <% end %> From 5e0918704d4198572c117ba2bec80286f9d4199e Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 11:21:36 -0500 Subject: [PATCH 14/17] trying to remove old values --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index a21791ec..4470eeab 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -32,6 +32,9 @@ match => { "message" => "^%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{SYSLOGPROG:program}%{SPACE}%{GREEDYDATA:message}$" } tag_on_failure => ["fail/s3-start/_grokparsefailure"] } + mutate { + remove_field => ["timestamp","program"] + } <% end %> <% if 'DEBUG' == p('logstash.metadata_level') %> From 6d7bb873893194a56419d073cdd19d3e3f44949a Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 11:28:59 -0500 Subject: [PATCH 15/17] trying to remove old values --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 4470eeab..9e8dea59 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -30,6 +30,7 @@ <% if !p('logstash_ingestor.cf-enabled') %> grok { match => { "message" => "^%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{SYSLOGPROG:program}%{SPACE}%{GREEDYDATA:message}$" } + overwrite => ["message"] tag_on_failure => ["fail/s3-start/_grokparsefailure"] } mutate { From cd9617eeae41d070ceb111733c453f8acadc769a Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 11:59:27 -0500 Subject: [PATCH 16/17] replacing message --- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 9e8dea59..4855ade9 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -29,12 +29,12 @@ <% if !p('logstash_ingestor.cf-enabled') %> grok { - match => { "message" => "^%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{SYSLOGPROG:program}%{SPACE}%{GREEDYDATA:message}$" } - overwrite => ["message"] + match => { "message" => "^%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{SYSLOGPROG:program}%{SPACE}%{GREEDYDATA:newmessage}$" } tag_on_failure => ["fail/s3-start/_grokparsefailure"] } mutate { remove_field => ["timestamp","program"] + replace => {"message" => "%{newmessage}" } } <% end %> From cb1a3f61a8abbf75fd00a0ee0ac4858eb18bcd56 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Tue, 19 Nov 2024 12:06:48 -0500 Subject: [PATCH 17/17] updated and working ingestion --- jobs/ingestor_syslog/spec | 4 ++-- jobs/ingestor_syslog/templates/bin/ingestor_syslog | 2 +- jobs/ingestor_syslog/templates/config/filters_pre.conf.erb | 2 +- .../templates/config/input_and_output.conf.erb | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec index 0fa49f4e..b88426b6 100644 --- a/jobs/ingestor_syslog/spec +++ b/jobs/ingestor_syslog/spec @@ -103,8 +103,8 @@ properties: description: The interval in milliseconds when a checkpoint is forced on the head page. default: 1000 - logstash_ingestor.cf-enabled: - description: are cf logs being ingested. + logstash_ingestor.ingest_syslog_enabled: + description: if True, logs are ingested from syslog default: true logstash_ingestor.filters: description: Filters to execute on the ingestors diff --git a/jobs/ingestor_syslog/templates/bin/ingestor_syslog b/jobs/ingestor_syslog/templates/bin/ingestor_syslog index 92f474af..2763e21c 100644 --- a/jobs/ingestor_syslog/templates/bin/ingestor_syslog +++ b/jobs/ingestor_syslog/templates/bin/ingestor_syslog @@ -92,7 +92,7 @@ cat ${JOB_DIR}/config/filters_post.conf >> ${JOB_DIR}/config/logstash.conf cat /var/vcap/packages/base-logstash-filters/if_it_looks_like_json.conf >> ${JOB_DIR}/config/logstash.conf <% end %> -<% if p('logstash_ingestor.cf-enabled') %> +<% if p('logstash_ingestor.ingest_syslog_enabled') %> cat /var/vcap/packages/base-logstash-filters/timecop.conf >> ${JOB_DIR}/config/logstash.conf <% end %> diff --git a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb index 4855ade9..a4be7733 100644 --- a/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_pre.conf.erb @@ -27,7 +27,7 @@ # - <% if !p('logstash_ingestor.cf-enabled') %> + <% if !p('logstash_ingestor.ingest_syslog_enabled') %> grok { match => { "message" => "^%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{SYSLOGPROG:program}%{SPACE}%{GREEDYDATA:newmessage}$" } tag_on_failure => ["fail/s3-start/_grokparsefailure"] diff --git a/jobs/ingestor_syslog/templates/config/input_and_output.conf.erb b/jobs/ingestor_syslog/templates/config/input_and_output.conf.erb index bdf04033..8a89e07b 100644 --- a/jobs/ingestor_syslog/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_syslog/templates/config/input_and_output.conf.erb @@ -1,7 +1,7 @@ input { - <% if p("logstash_ingestor.cf-enabled") %> + <% if p("logstash_ingestor.ingest_syslog_enabled") %> <% if_p("logstash_ingestor.syslog.port") do |port| %> tcp {