From f0ae2e898f3ce67ddfab179cfcea273bfa8b941e Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Wed, 18 Sep 2024 12:51:33 -0400 Subject: [PATCH 1/8] adding in filtering --- .../templates/config/input_and_output.conf.erb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index ea934a14..4495e28c 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -12,6 +12,22 @@ input } } +filter +{ + mutate { + rename => {"[cloudwatch_logs][tags][environment]"=>"environment"} + rename => {"[cloudwatch_logs][tags][OrganizationGUID]"=>"[@cf][org_id]"} + rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"} + rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} + rename => {"[cloudwatch_logs][tags][Createdat]"=>"Created_at"} + rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"instance_id"} + rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service"} + rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"plan"} + rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} + rename => {"[cloudwatch_logs][tags][client]"=>"client"} + + } +} <% es_output_hosts = nil From 3bb2e79cd565373c375252c060bc6f5eea3652ec Mon Sep 17 00:00:00 2001 From: Jason Gambino Date: Wed, 18 Sep 2024 14:26:33 -0400 Subject: [PATCH 2/8] Update jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb Co-authored-by: Mark Boyd Signed-off-by: Jason Gambino --- .../templates/config/input_and_output.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index 4495e28c..74b713b6 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -22,7 +22,7 @@ filter rename => {"[cloudwatch_logs][tags][Createdat]"=>"Created_at"} rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"instance_id"} rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service"} - rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"plan"} + rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"service_plan"} rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} rename => {"[cloudwatch_logs][tags][client]"=>"client"} From 5fae4cc612139133348c2d9877ce141f4180c2fd Mon Sep 17 00:00:00 2001 From: Jason Gambino Date: Wed, 18 Sep 2024 14:26:41 -0400 Subject: [PATCH 3/8] Update jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb Co-authored-by: Mark Boyd Signed-off-by: Jason Gambino --- .../templates/config/input_and_output.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index 74b713b6..4c822793 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -20,7 +20,7 @@ filter rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"} rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} rename => {"[cloudwatch_logs][tags][Createdat]"=>"Created_at"} - rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"instance_id"} + rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"service_instance_id"} rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service"} rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"service_plan"} rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} From c9d1d016a597ed5a9f2e847d253f3cb8ae7ff64a Mon Sep 17 00:00:00 2001 From: Jason Gambino Date: Wed, 18 Sep 2024 14:26:47 -0400 Subject: [PATCH 4/8] Update jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb Co-authored-by: Mark Boyd Signed-off-by: Jason Gambino --- .../templates/config/input_and_output.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index 4c822793..a123f310 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -21,7 +21,7 @@ filter rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} rename => {"[cloudwatch_logs][tags][Createdat]"=>"Created_at"} rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"service_instance_id"} - rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service"} + rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service_offering"} rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"service_plan"} rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} rename => {"[cloudwatch_logs][tags][client]"=>"client"} From e7f2eec1326e344970a8e4bdc36ef9ff48aa9659 Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Wed, 18 Sep 2024 14:49:16 -0400 Subject: [PATCH 5/8] removing unneeded field --- .../templates/config/input_and_output.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index a123f310..b87b784a 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -19,7 +19,7 @@ filter rename => {"[cloudwatch_logs][tags][OrganizationGUID]"=>"[@cf][org_id]"} rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"} rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} - rename => {"[cloudwatch_logs][tags][Createdat]"=>"Created_at"} + remove_field => ["[cloudwatch_logs][tags][Createdat]"] rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"service_instance_id"} rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service_offering"} rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"service_plan"} From fb3c1c034201ac2b714996e1570c602d24e0661b Mon Sep 17 00:00:00 2001 From: "Jason A. Gambino" Date: Wed, 18 Sep 2024 14:50:28 -0400 Subject: [PATCH 6/8] removing unneeded field --- .../templates/config/input_and_output.conf.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index b87b784a..f1e2ae01 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -20,6 +20,7 @@ filter rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"} rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} remove_field => ["[cloudwatch_logs][tags][Createdat]"] + remove_field => ["[cloudwatch_logs][tags][Updatedat]"] rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"service_instance_id"} rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service_offering"} rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"service_plan"} From f400193d39649008cd6ad9dce1c0e880b58d3f07 Mon Sep 17 00:00:00 2001 From: Jason Gambino Date: Wed, 18 Sep 2024 14:57:31 -0400 Subject: [PATCH 7/8] Update jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb Co-authored-by: Mark Boyd Signed-off-by: Jason Gambino --- .../templates/config/input_and_output.conf.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index f1e2ae01..ac6d540f 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -21,9 +21,9 @@ filter rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} remove_field => ["[cloudwatch_logs][tags][Createdat]"] remove_field => ["[cloudwatch_logs][tags][Updatedat]"] - rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"service_instance_id"} - rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"service_offering"} - rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"service_plan"} + rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"[@cf][service_instance_id]"} + rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"[@cf][service_offering]"} + rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"[@cf][service_plan]"} rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} rename => {"[cloudwatch_logs][tags][client]"=>"client"} From 609b81c6fca5daa223d5ffb3008303db33d7bb4f Mon Sep 17 00:00:00 2001 From: Jason Gambino Date: Wed, 18 Sep 2024 14:57:43 -0400 Subject: [PATCH 8/8] Update jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb Co-authored-by: Mark Boyd Signed-off-by: Jason Gambino --- .../templates/config/input_and_output.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index ac6d540f..181807b8 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -25,7 +25,7 @@ filter rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"[@cf][service_offering]"} rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"[@cf][service_plan]"} rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} - rename => {"[cloudwatch_logs][tags][client]"=>"client"} + remove_field => ["[cloudwatch_logs][tags][client]"] } }