(CLOUD-179) Terminate run on inconsistent AWS data

If we query AWS for a resource (by id) that doesn't exist we will raise an exception. Due to PUP-3656 that prevents prefetch but lets the provider continue with incorrect data. This changes that behaviour to stop the provider run if we discover inconsistent data. It also provides information to help debug the problem - although in many cases running a second time will fix the issue. Common causes of inconsistency are: * eventual consistency / propagation of resource data within AWS * other clients acting on the API at the same time This change also allows rate-limiting errors to immediately cause the run to stop. Paired-with: <[email protected]> Paired-with: <[email protected]> Paired-with: <[email protected]>
puppetlabs-toy-chest · Mar 24, 2015 · ab2b948 · ab2b948
1 parent 0e2ff12
commit ab2b948
Show file tree

Hide file tree

Showing 18 changed files with 221 additions and 131 deletions.
diff --git a/lib/puppet/provider/cloudwatch_alarm/v2.rb b/lib/puppet/provider/cloudwatch_alarm/v2.rb
@@ -7,14 +7,18 @@
 
   def self.instances
     regions.collect do |region|
-      alarms = []
-      cloudwatch_client(region).describe_alarms.each do |response|
-        response.data.metric_alarms.each do |alarm|
-          hash = alarm_to_hash(region, alarm)
-          alarms << new(hash)
+      begin
+        alarms = []
+        cloudwatch_client(region).describe_alarms.each do |response|
+          response.data.metric_alarms.each do |alarm|
+            hash = alarm_to_hash(region, alarm)
+            alarms << new(hash)
+          end
         end
+        alarms
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      alarms
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_autoscalinggroup/v2.rb b/lib/puppet/provider/ec2_autoscalinggroup/v2.rb
@@ -7,14 +7,18 @@
 
   def self.instances
     regions.collect do |region|
-      groups = []
-      autoscaling_client(region).describe_auto_scaling_groups.each do |response|
-        response.data.auto_scaling_groups.each do |group|
-          hash = group_to_hash(region, group)
-          groups << new(hash)
+      begin
+        groups = []
+        autoscaling_client(region).describe_auto_scaling_groups.each do |response|
+          response.data.auto_scaling_groups.each do |group|
+            hash = group_to_hash(region, group)
+            groups << new(hash)
+          end
         end
+        groups
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      groups
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_elastic_ip/v2.rb b/lib/puppet/provider/ec2_elastic_ip/v2.rb
@@ -7,30 +7,34 @@
 
   def self.instances
     regions.collect do |region|
-      ec2 = ec2_client(region)
-      ec2.describe_addresses.addresses.collect do |address|
-        instance_name = nil
-        unless address.instance_id.nil? || address.instance_id.empty?
-          instances = ec2.describe_instances(instance_ids: [address.instance_id]).collect do |response|
-            response.data.reservations.collect do |reservation|
-              reservation.instances.collect do |instance|
-                instance
-              end
+      begin
+        ec2 = ec2_client(region)
+        ec2.describe_addresses.addresses.collect do |address|
+          instance_name = nil
+          unless address.instance_id.nil? || address.instance_id.empty?
+            instances = ec2.describe_instances(instance_ids: [address.instance_id]).collect do |response|
+              response.data.reservations.collect do |reservation|
+                reservation.instances.collect do |instance|
+                  instance
+                end
+              end.flatten
             end.flatten
-          end.flatten
-          name_tag = instances.first.tags.detect { |tag| tag.key == 'Name' }
-          instance_name = name_tag ? name_tag.value : nil
+            name_tag = instances.first.tags.detect { |tag| tag.key == 'Name' }
+            instance_name = name_tag ? name_tag.value : nil
+          end
+          new({
+            name: address.public_ip,
+            instance_id: address.instance_id,
+            instance: instance_name,
+            allocation_id: address.allocation_id,
+            association_id: address.association_id,
+            domain: address.domain,
+            ensure: instance_name ? :attached : :detached,
+            region: region,
+          })
         end
-        new({
-          name: address.public_ip,
-          instance_id: address.instance_id,
-          instance: instance_name,
-          allocation_id: address.allocation_id,
-          association_id: address.association_id,
-          domain: address.domain,
-          ensure: instance_name ? :attached : :detached,
-          region: region,
-        })
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
     end.flatten
   end

diff --git a/lib/puppet/provider/ec2_instance/v2.rb b/lib/puppet/provider/ec2_instance/v2.rb
@@ -9,18 +9,22 @@
 
   def self.instances
     regions.collect do |region|
-      instances = []
-      ec2_client(region).describe_instances(filters: [
-        {name: 'instance-state-name', values: ['pending', 'running', 'stopping', 'stopped']}
-      ]).each do |response|
-        response.data.reservations.each do |reservation|
-          reservation.instances.each do |instance|
-            hash = instance_to_hash(region, instance)
-            instances << new(hash) if has_name?(hash)
+      begin
+        instances = []
+        ec2_client(region).describe_instances(filters: [
+          {name: 'instance-state-name', values: ['pending', 'running', 'stopping', 'stopped']}
+        ]).each do |response|
+          response.data.reservations.each do |reservation|
+            reservation.instances.each do |instance|
+              hash = instance_to_hash(region, instance)
+              instances << new(hash) if has_name?(hash)
+            end
           end
         end
+        instances
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      instances
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_launchconfiguration/v2.rb b/lib/puppet/provider/ec2_launchconfiguration/v2.rb
@@ -8,14 +8,18 @@
 
   def self.instances
     regions.collect do |region|
-      launch_configs = []
-      autoscaling_client(region).describe_launch_configurations.each do |response|
-        response.data.launch_configurations.each do |config|
-          hash = config_to_hash(region, config)
-          launch_configs << new(hash)
+      begin
+        launch_configs = []
+        autoscaling_client(region).describe_launch_configurations.each do |response|
+          response.data.launch_configurations.each do |config|
+            hash = config_to_hash(region, config)
+            launch_configs << new(hash)
+          end
         end
+        launch_configs
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      launch_configs
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_scalingpolicy/v2.rb b/lib/puppet/provider/ec2_scalingpolicy/v2.rb
@@ -7,14 +7,18 @@
 
   def self.instances
     regions.collect do |region|
-      policies = []
-      autoscaling_client(region).describe_policies.each do |response|
-        response.data.scaling_policies.each do |policy|
-          hash = policy_to_hash(region, policy)
-          policies << new(hash)
+      begin
+        policies = []
+        autoscaling_client(region).describe_policies.each do |response|
+          response.data.scaling_policies.each do |policy|
+            hash = policy_to_hash(region, policy)
+            policies << new(hash)
+          end
         end
+        policies
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      policies
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_securitygroup/v2.rb b/lib/puppet/provider/ec2_securitygroup/v2.rb
@@ -9,13 +9,17 @@
 
   def self.instances
     regions.collect do |region|
-      groups = []
-      ec2_client(region).describe_security_groups.each do |response|
-        response.data.security_groups.collect do |group|
-          groups << new(security_group_to_hash(region, group))
+      begin
+        groups = []
+        ec2_client(region).describe_security_groups.each do |response|
+          response.data.security_groups.collect do |group|
+            groups << new(security_group_to_hash(region, group))
+          end
         end
+        groups
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      groups
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_vpc/v2.rb b/lib/puppet/provider/ec2_vpc/v2.rb
@@ -9,13 +9,17 @@
 
   def self.instances
     regions.collect do |region|
-      response = ec2_client(region).describe_vpcs()
-      vpcs = []
-      response.data.vpcs.each do |vpc|
-        hash = vpc_to_hash(region, vpc)
-        vpcs << new(hash) if has_name?(hash)
+      begin
+        response = ec2_client(region).describe_vpcs()
+        vpcs = []
+        response.data.vpcs.each do |vpc|
+          hash = vpc_to_hash(region, vpc)
+          vpcs << new(hash) if has_name?(hash)
+        end
+        vpcs
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      vpcs
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_vpc_customer_gateway/v2.rb b/lib/puppet/provider/ec2_vpc_customer_gateway/v2.rb
@@ -9,14 +9,18 @@
 
   def self.instances()
     regions.collect do |region|
-      gateways = []
-      ec2_client(region).describe_customer_gateways.each do |response|
-        response.data.customer_gateways.each do |gateway|
-          hash = gateway_to_hash(region, gateway)
-          gateways << new(hash) unless (gateway.state == "deleting" or gateway.state == "deleted")
+      begin
+        gateways = []
+        ec2_client(region).describe_customer_gateways.each do |response|
+          response.data.customer_gateways.each do |gateway|
+            hash = gateway_to_hash(region, gateway)
+            gateways << new(hash) unless (gateway.state == "deleting" or gateway.state == "deleted")
+          end
         end
+        gateways
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      gateways
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_vpc_dhcp_options/v2.rb b/lib/puppet/provider/ec2_vpc_dhcp_options/v2.rb
@@ -9,14 +9,18 @@
 
   def self.instances
     regions.collect do |region|
-      options = []
-      ec2_client(region).describe_dhcp_options.collect do |response|
-        response.data.dhcp_options.each do |item|
-          hash = dhcp_option_to_hash(region, item)
-          options << new(hash) if has_name?(hash)
+      begin
+        options = []
+        ec2_client(region).describe_dhcp_options.collect do |response|
+          response.data.dhcp_options.each do |item|
+            hash = dhcp_option_to_hash(region, item)
+            options << new(hash) if has_name?(hash)
+          end
         end
+        options
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      options
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_vpc_internet_gateway/v2.rb b/lib/puppet/provider/ec2_vpc_internet_gateway/v2.rb
@@ -9,13 +9,17 @@
 
   def self.instances
     regions.collect do |region|
-      response = ec2_client(region).describe_internet_gateways()
-      gateways = []
-      response.data.internet_gateways.each do |gateway|
-        hash = gateway_to_hash(region, gateway)
-        gateways << new(hash) if has_name?(hash)
+      begin
+        response = ec2_client(region).describe_internet_gateways()
+        gateways = []
+        response.data.internet_gateways.each do |gateway|
+          hash = gateway_to_hash(region, gateway)
+          gateways << new(hash) if has_name?(hash)
+        end
+        gateways
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      gateways
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_vpc_routetable/v2.rb b/lib/puppet/provider/ec2_vpc_routetable/v2.rb
@@ -9,13 +9,17 @@
 
   def self.instances
     regions.collect do |region|
-      response = ec2_client(region).describe_route_tables()
-      tables = []
-      response.data.route_tables.each do |table|
-        hash = route_table_to_hash(region, table)
-        tables << new(hash) if has_name?(hash)
+      begin
+        response = ec2_client(region).describe_route_tables()
+        tables = []
+        response.data.route_tables.each do |table|
+          hash = route_table_to_hash(region, table)
+          tables << new(hash) if has_name?(hash)
+        end
+        tables
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      tables
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_vpc_subnet/v2.rb b/lib/puppet/provider/ec2_vpc_subnet/v2.rb
@@ -9,13 +9,17 @@
 
   def self.instances
     regions.collect do |region|
-      response = ec2_client(region).describe_subnets()
-      subnets = []
-      response.data.subnets.each do |subnet|
-        hash = subnet_to_hash(region, subnet)
-        subnets << new(hash) if has_name?(hash)
+      begin
+        response = ec2_client(region).describe_subnets()
+        subnets = []
+        response.data.subnets.each do |subnet|
+          hash = subnet_to_hash(region, subnet)
+          subnets << new(hash) if has_name?(hash)
+        end
+        subnets
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      subnets
     end.flatten
   end
 

diff --git a/lib/puppet/provider/ec2_vpc_vpn/v2.rb b/lib/puppet/provider/ec2_vpc_vpn/v2.rb
@@ -9,16 +9,20 @@
 
   def self.instances()
     regions.collect do |region|
-      connections = []
-      ec2_client(region).describe_vpn_connections(filters: [
-        {:name => 'state', :values => ['pending', 'available']}
-      ]).each do |response|
-        response.data.vpn_connections.each do |connection|
-          hash = connection_to_hash(region, connection)
-          connections << new(hash) if has_name?(hash)
+      begin
+        connections = []
+        ec2_client(region).describe_vpn_connections(filters: [
+          {:name => 'state', :values => ['pending', 'available']}
+        ]).each do |response|
+          response.data.vpn_connections.each do |connection|
+            hash = connection_to_hash(region, connection)
+            connections << new(hash) if has_name?(hash)
+          end
         end
+        connections
+      rescue StandardError => e
+        raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message)
       end
-      connections
     end.flatten
   end