Merge pull request #174 from cloud-gov/RDS-ingestor-cleanup

Rds ingestor cleanup
cloud-gov · Dec 20, 2024 · 262280a · 262280a
2 parents 577a418 + 2a76660
commit 262280a
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 16 deletions.
diff --git a/jobs/ingestor_cloudwatch/templates/bin/ingestor_cloudwatch b/jobs/ingestor_cloudwatch/templates/bin/ingestor_cloudwatch
@@ -46,8 +46,6 @@ export LOGSTASH_WORKERS=`grep -c ^processor /proc/cpuinfo`
 <% else %>
 export LOGSTASH_WORKERS=<%= p('logstash_parser.workers') %>
 <% end %>
-export TIMECOP_REJECT_GREATER_THAN_HOURS=<%= p('logstash_parser.timecop.reject_greater_than_hours') %>
-export TIMECOP_REJECT_LESS_THAN_HOURS=<%= p('logstash_parser.timecop.reject_less_than_hours') %>
 export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }' ) * <%= p("logstash.heap_percentage") %> ) / 100 ))K
 <% if_p('logstash.heap_size') do |heap_size| %>
 HEAP_SIZE=<%= heap_size %>
@@ -56,12 +54,6 @@ HEAP_SIZE=<%= heap_size %>
 export <%= env.keys[0] %>="<%= env.values[0] %>"
 <% end %>
 
-
-# These are what changes between ingestors
-<% p("logstash_parser.wait_for_templates").each do |template| %>
-wait_for_template "<%= template %>"
-<% end %>
-
 export LS_JAVA_OPTS="-Xms$HEAP_SIZE -Xmx$HEAP_SIZE -DPID=$$"
 
 # construct a complete config file from all the fragments

diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb
@@ -21,28 +21,25 @@ filter
 
         rename => {"[cloudwatch_logs][tags][OrganizationGUID]"=>"[@cf][org_id]"}
         rename => {"[cloudwatch_logs][tags][Organization GUID]"=>"[@cf][org_id]"}
+        rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"}
+        rename => {"[cloudwatch_logs][tags][Organization name]"=>"[@cf][org]"}
 
         rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"}
         rename => {"[cloudwatch_logs][tags][Space GUID]"=>"[@cf][space_id]"}
-
         rename => {"[cloudwatch_logs][tags][Spacename]"=>"[@cf][space]"}
         rename => {"[cloudwatch_logs][tags][Space name]"=>"[@cf][space]"}
 
-        rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"}
-        rename => {"[cloudwatch_logs][tags][Organization name]"=>"[@cf][org]"}
-
         rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"[@cf][service_instance_id]"}
         rename => {"[cloudwatch_logs][tags][Instance GUID]"=>"[@cf][service_instance_id]"}
-
         rename => {"[cloudwatch_logs][tags][Instance name]"=>"[@cf][service]"}
 
         rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"[@cf][service_offering]"}
         rename => {"[cloudwatch_logs][tags][Service offering name]"=>"[@cf][service_offering]"}
-
         rename => {"[cloudwatch_logs][tags][Service plan name]"=>"[@cf][service_plan]"}
-
         rename => {"[cloudwatch_logs][tags][service]"=>"broker"}
+
         rename => {"[cloudwatch_logs][tags][broker]"=>"broker"}
+        rename => {"message" => "@message" }
 
         remove_field => ["[cloudwatch_logs][tags][Createdat]"]
         remove_field => ["[cloudwatch_logs][tags][Created at]"]
@@ -54,7 +51,6 @@ filter
         remove_field => ["[cloudwatch_logs][tags][ServiceGUID]"]
         remove_field => ["[cloudwatch_logs][tags][Service GUID]"]
 
-        rename => [ "message" => "@message" ]
     }
     truncate {
       fields => ["@message"]