From ab2b948c24bd386b246f3a61b45776db786edc34 Mon Sep 17 00:00:00 2001 From: Gareth Rushgrove Date: Mon, 23 Mar 2015 09:31:30 +0000 Subject: [PATCH] (CLOUD-179) Terminate run on inconsistent AWS data If we query AWS for a resource (by id) that doesn't exist we will raise an exception. Due to PUP-3656 that prevents prefetch but lets the provider continue with incorrect data. This changes that behaviour to stop the provider run if we discover inconsistent data. It also provides information to help debug the problem - although in many cases running a second time will fix the issue. Common causes of inconsistency are: * eventual consistency / propagation of resource data within AWS * other clients acting on the API at the same time This change also allows rate-limiting errors to immediately cause the run to stop. Paired-with: Paired-with: Paired-with: --- lib/puppet/provider/cloudwatch_alarm/v2.rb | 16 ++++--- .../provider/ec2_autoscalinggroup/v2.rb | 16 ++++--- lib/puppet/provider/ec2_elastic_ip/v2.rb | 48 ++++++++++--------- lib/puppet/provider/ec2_instance/v2.rb | 22 +++++---- .../provider/ec2_launchconfiguration/v2.rb | 16 ++++--- lib/puppet/provider/ec2_scalingpolicy/v2.rb | 16 ++++--- lib/puppet/provider/ec2_securitygroup/v2.rb | 14 ++++-- lib/puppet/provider/ec2_vpc/v2.rb | 16 ++++--- .../provider/ec2_vpc_customer_gateway/v2.rb | 16 ++++--- .../provider/ec2_vpc_dhcp_options/v2.rb | 16 ++++--- .../provider/ec2_vpc_internet_gateway/v2.rb | 16 ++++--- lib/puppet/provider/ec2_vpc_routetable/v2.rb | 16 ++++--- lib/puppet/provider/ec2_vpc_subnet/v2.rb | 16 ++++--- lib/puppet/provider/ec2_vpc_vpn/v2.rb | 20 ++++---- lib/puppet/provider/ec2_vpc_vpn_gateway/v2.rb | 20 ++++---- lib/puppet/provider/elb_loadbalancer/v2.rb | 16 ++++--- lib/puppet/provider/route53_record.rb | 30 +++++++----- lib/puppet_x/puppetlabs/aws.rb | 22 +++++++++ 18 files changed, 221 insertions(+), 131 deletions(-) diff --git a/lib/puppet/provider/cloudwatch_alarm/v2.rb b/lib/puppet/provider/cloudwatch_alarm/v2.rb index 2ae3f5cf..500d3123 100644 --- a/lib/puppet/provider/cloudwatch_alarm/v2.rb +++ b/lib/puppet/provider/cloudwatch_alarm/v2.rb @@ -7,14 +7,18 @@ def self.instances regions.collect do |region| - alarms = [] - cloudwatch_client(region).describe_alarms.each do |response| - response.data.metric_alarms.each do |alarm| - hash = alarm_to_hash(region, alarm) - alarms << new(hash) + begin + alarms = [] + cloudwatch_client(region).describe_alarms.each do |response| + response.data.metric_alarms.each do |alarm| + hash = alarm_to_hash(region, alarm) + alarms << new(hash) + end end + alarms + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - alarms end.flatten end diff --git a/lib/puppet/provider/ec2_autoscalinggroup/v2.rb b/lib/puppet/provider/ec2_autoscalinggroup/v2.rb index 9f435440..a050f4bf 100644 --- a/lib/puppet/provider/ec2_autoscalinggroup/v2.rb +++ b/lib/puppet/provider/ec2_autoscalinggroup/v2.rb @@ -7,14 +7,18 @@ def self.instances regions.collect do |region| - groups = [] - autoscaling_client(region).describe_auto_scaling_groups.each do |response| - response.data.auto_scaling_groups.each do |group| - hash = group_to_hash(region, group) - groups << new(hash) + begin + groups = [] + autoscaling_client(region).describe_auto_scaling_groups.each do |response| + response.data.auto_scaling_groups.each do |group| + hash = group_to_hash(region, group) + groups << new(hash) + end end + groups + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - groups end.flatten end diff --git a/lib/puppet/provider/ec2_elastic_ip/v2.rb b/lib/puppet/provider/ec2_elastic_ip/v2.rb index bb6aaf51..68c4a049 100644 --- a/lib/puppet/provider/ec2_elastic_ip/v2.rb +++ b/lib/puppet/provider/ec2_elastic_ip/v2.rb @@ -7,30 +7,34 @@ def self.instances regions.collect do |region| - ec2 = ec2_client(region) - ec2.describe_addresses.addresses.collect do |address| - instance_name = nil - unless address.instance_id.nil? || address.instance_id.empty? - instances = ec2.describe_instances(instance_ids: [address.instance_id]).collect do |response| - response.data.reservations.collect do |reservation| - reservation.instances.collect do |instance| - instance - end + begin + ec2 = ec2_client(region) + ec2.describe_addresses.addresses.collect do |address| + instance_name = nil + unless address.instance_id.nil? || address.instance_id.empty? + instances = ec2.describe_instances(instance_ids: [address.instance_id]).collect do |response| + response.data.reservations.collect do |reservation| + reservation.instances.collect do |instance| + instance + end + end.flatten end.flatten - end.flatten - name_tag = instances.first.tags.detect { |tag| tag.key == 'Name' } - instance_name = name_tag ? name_tag.value : nil + name_tag = instances.first.tags.detect { |tag| tag.key == 'Name' } + instance_name = name_tag ? name_tag.value : nil + end + new({ + name: address.public_ip, + instance_id: address.instance_id, + instance: instance_name, + allocation_id: address.allocation_id, + association_id: address.association_id, + domain: address.domain, + ensure: instance_name ? :attached : :detached, + region: region, + }) end - new({ - name: address.public_ip, - instance_id: address.instance_id, - instance: instance_name, - allocation_id: address.allocation_id, - association_id: address.association_id, - domain: address.domain, - ensure: instance_name ? :attached : :detached, - region: region, - }) + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end end.flatten end diff --git a/lib/puppet/provider/ec2_instance/v2.rb b/lib/puppet/provider/ec2_instance/v2.rb index 45ef0b9d..339d6947 100644 --- a/lib/puppet/provider/ec2_instance/v2.rb +++ b/lib/puppet/provider/ec2_instance/v2.rb @@ -9,18 +9,22 @@ def self.instances regions.collect do |region| - instances = [] - ec2_client(region).describe_instances(filters: [ - {name: 'instance-state-name', values: ['pending', 'running', 'stopping', 'stopped']} - ]).each do |response| - response.data.reservations.each do |reservation| - reservation.instances.each do |instance| - hash = instance_to_hash(region, instance) - instances << new(hash) if has_name?(hash) + begin + instances = [] + ec2_client(region).describe_instances(filters: [ + {name: 'instance-state-name', values: ['pending', 'running', 'stopping', 'stopped']} + ]).each do |response| + response.data.reservations.each do |reservation| + reservation.instances.each do |instance| + hash = instance_to_hash(region, instance) + instances << new(hash) if has_name?(hash) + end end end + instances + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - instances end.flatten end diff --git a/lib/puppet/provider/ec2_launchconfiguration/v2.rb b/lib/puppet/provider/ec2_launchconfiguration/v2.rb index f6c4caa5..4383fa34 100644 --- a/lib/puppet/provider/ec2_launchconfiguration/v2.rb +++ b/lib/puppet/provider/ec2_launchconfiguration/v2.rb @@ -8,14 +8,18 @@ def self.instances regions.collect do |region| - launch_configs = [] - autoscaling_client(region).describe_launch_configurations.each do |response| - response.data.launch_configurations.each do |config| - hash = config_to_hash(region, config) - launch_configs << new(hash) + begin + launch_configs = [] + autoscaling_client(region).describe_launch_configurations.each do |response| + response.data.launch_configurations.each do |config| + hash = config_to_hash(region, config) + launch_configs << new(hash) + end end + launch_configs + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - launch_configs end.flatten end diff --git a/lib/puppet/provider/ec2_scalingpolicy/v2.rb b/lib/puppet/provider/ec2_scalingpolicy/v2.rb index 7eab1012..4e91bc8c 100644 --- a/lib/puppet/provider/ec2_scalingpolicy/v2.rb +++ b/lib/puppet/provider/ec2_scalingpolicy/v2.rb @@ -7,14 +7,18 @@ def self.instances regions.collect do |region| - policies = [] - autoscaling_client(region).describe_policies.each do |response| - response.data.scaling_policies.each do |policy| - hash = policy_to_hash(region, policy) - policies << new(hash) + begin + policies = [] + autoscaling_client(region).describe_policies.each do |response| + response.data.scaling_policies.each do |policy| + hash = policy_to_hash(region, policy) + policies << new(hash) + end end + policies + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - policies end.flatten end diff --git a/lib/puppet/provider/ec2_securitygroup/v2.rb b/lib/puppet/provider/ec2_securitygroup/v2.rb index 4e7eac34..a410c73d 100644 --- a/lib/puppet/provider/ec2_securitygroup/v2.rb +++ b/lib/puppet/provider/ec2_securitygroup/v2.rb @@ -9,13 +9,17 @@ def self.instances regions.collect do |region| - groups = [] - ec2_client(region).describe_security_groups.each do |response| - response.data.security_groups.collect do |group| - groups << new(security_group_to_hash(region, group)) + begin + groups = [] + ec2_client(region).describe_security_groups.each do |response| + response.data.security_groups.collect do |group| + groups << new(security_group_to_hash(region, group)) + end end + groups + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - groups end.flatten end diff --git a/lib/puppet/provider/ec2_vpc/v2.rb b/lib/puppet/provider/ec2_vpc/v2.rb index f52bebf1..8efa1f98 100644 --- a/lib/puppet/provider/ec2_vpc/v2.rb +++ b/lib/puppet/provider/ec2_vpc/v2.rb @@ -9,13 +9,17 @@ def self.instances regions.collect do |region| - response = ec2_client(region).describe_vpcs() - vpcs = [] - response.data.vpcs.each do |vpc| - hash = vpc_to_hash(region, vpc) - vpcs << new(hash) if has_name?(hash) + begin + response = ec2_client(region).describe_vpcs() + vpcs = [] + response.data.vpcs.each do |vpc| + hash = vpc_to_hash(region, vpc) + vpcs << new(hash) if has_name?(hash) + end + vpcs + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - vpcs end.flatten end diff --git a/lib/puppet/provider/ec2_vpc_customer_gateway/v2.rb b/lib/puppet/provider/ec2_vpc_customer_gateway/v2.rb index 2b6802c2..0eba8a8a 100644 --- a/lib/puppet/provider/ec2_vpc_customer_gateway/v2.rb +++ b/lib/puppet/provider/ec2_vpc_customer_gateway/v2.rb @@ -9,14 +9,18 @@ def self.instances() regions.collect do |region| - gateways = [] - ec2_client(region).describe_customer_gateways.each do |response| - response.data.customer_gateways.each do |gateway| - hash = gateway_to_hash(region, gateway) - gateways << new(hash) unless (gateway.state == "deleting" or gateway.state == "deleted") + begin + gateways = [] + ec2_client(region).describe_customer_gateways.each do |response| + response.data.customer_gateways.each do |gateway| + hash = gateway_to_hash(region, gateway) + gateways << new(hash) unless (gateway.state == "deleting" or gateway.state == "deleted") + end end + gateways + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - gateways end.flatten end diff --git a/lib/puppet/provider/ec2_vpc_dhcp_options/v2.rb b/lib/puppet/provider/ec2_vpc_dhcp_options/v2.rb index e6c15689..5b208bd0 100644 --- a/lib/puppet/provider/ec2_vpc_dhcp_options/v2.rb +++ b/lib/puppet/provider/ec2_vpc_dhcp_options/v2.rb @@ -9,14 +9,18 @@ def self.instances regions.collect do |region| - options = [] - ec2_client(region).describe_dhcp_options.collect do |response| - response.data.dhcp_options.each do |item| - hash = dhcp_option_to_hash(region, item) - options << new(hash) if has_name?(hash) + begin + options = [] + ec2_client(region).describe_dhcp_options.collect do |response| + response.data.dhcp_options.each do |item| + hash = dhcp_option_to_hash(region, item) + options << new(hash) if has_name?(hash) + end end + options + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - options end.flatten end diff --git a/lib/puppet/provider/ec2_vpc_internet_gateway/v2.rb b/lib/puppet/provider/ec2_vpc_internet_gateway/v2.rb index 0a8b170e..a0b5759f 100644 --- a/lib/puppet/provider/ec2_vpc_internet_gateway/v2.rb +++ b/lib/puppet/provider/ec2_vpc_internet_gateway/v2.rb @@ -9,13 +9,17 @@ def self.instances regions.collect do |region| - response = ec2_client(region).describe_internet_gateways() - gateways = [] - response.data.internet_gateways.each do |gateway| - hash = gateway_to_hash(region, gateway) - gateways << new(hash) if has_name?(hash) + begin + response = ec2_client(region).describe_internet_gateways() + gateways = [] + response.data.internet_gateways.each do |gateway| + hash = gateway_to_hash(region, gateway) + gateways << new(hash) if has_name?(hash) + end + gateways + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - gateways end.flatten end diff --git a/lib/puppet/provider/ec2_vpc_routetable/v2.rb b/lib/puppet/provider/ec2_vpc_routetable/v2.rb index 74376fe4..cf92e218 100644 --- a/lib/puppet/provider/ec2_vpc_routetable/v2.rb +++ b/lib/puppet/provider/ec2_vpc_routetable/v2.rb @@ -9,13 +9,17 @@ def self.instances regions.collect do |region| - response = ec2_client(region).describe_route_tables() - tables = [] - response.data.route_tables.each do |table| - hash = route_table_to_hash(region, table) - tables << new(hash) if has_name?(hash) + begin + response = ec2_client(region).describe_route_tables() + tables = [] + response.data.route_tables.each do |table| + hash = route_table_to_hash(region, table) + tables << new(hash) if has_name?(hash) + end + tables + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - tables end.flatten end diff --git a/lib/puppet/provider/ec2_vpc_subnet/v2.rb b/lib/puppet/provider/ec2_vpc_subnet/v2.rb index 666b5cff..38e2674d 100644 --- a/lib/puppet/provider/ec2_vpc_subnet/v2.rb +++ b/lib/puppet/provider/ec2_vpc_subnet/v2.rb @@ -9,13 +9,17 @@ def self.instances regions.collect do |region| - response = ec2_client(region).describe_subnets() - subnets = [] - response.data.subnets.each do |subnet| - hash = subnet_to_hash(region, subnet) - subnets << new(hash) if has_name?(hash) + begin + response = ec2_client(region).describe_subnets() + subnets = [] + response.data.subnets.each do |subnet| + hash = subnet_to_hash(region, subnet) + subnets << new(hash) if has_name?(hash) + end + subnets + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - subnets end.flatten end diff --git a/lib/puppet/provider/ec2_vpc_vpn/v2.rb b/lib/puppet/provider/ec2_vpc_vpn/v2.rb index e2907f18..3cca0016 100644 --- a/lib/puppet/provider/ec2_vpc_vpn/v2.rb +++ b/lib/puppet/provider/ec2_vpc_vpn/v2.rb @@ -9,16 +9,20 @@ def self.instances() regions.collect do |region| - connections = [] - ec2_client(region).describe_vpn_connections(filters: [ - {:name => 'state', :values => ['pending', 'available']} - ]).each do |response| - response.data.vpn_connections.each do |connection| - hash = connection_to_hash(region, connection) - connections << new(hash) if has_name?(hash) + begin + connections = [] + ec2_client(region).describe_vpn_connections(filters: [ + {:name => 'state', :values => ['pending', 'available']} + ]).each do |response| + response.data.vpn_connections.each do |connection| + hash = connection_to_hash(region, connection) + connections << new(hash) if has_name?(hash) + end end + connections + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - connections end.flatten end diff --git a/lib/puppet/provider/ec2_vpc_vpn_gateway/v2.rb b/lib/puppet/provider/ec2_vpc_vpn_gateway/v2.rb index c8410dac..df01cf64 100644 --- a/lib/puppet/provider/ec2_vpc_vpn_gateway/v2.rb +++ b/lib/puppet/provider/ec2_vpc_vpn_gateway/v2.rb @@ -9,16 +9,20 @@ def self.instances() regions.collect do |region| - gateways = [] - ec2_client(region).describe_vpn_gateways(filters: [ - {:name => 'state', :values => ['pending', 'available']} - ]).each do |response| - response.data.vpn_gateways.each do |gateway| - hash = gateway_to_hash(region, gateway) - gateways << new(hash) if has_name?(hash) + begin + gateways = [] + ec2_client(region).describe_vpn_gateways(filters: [ + {:name => 'state', :values => ['pending', 'available']} + ]).each do |response| + response.data.vpn_gateways.each do |gateway| + hash = gateway_to_hash(region, gateway) + gateways << new(hash) if has_name?(hash) + end end + gateways + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - gateways end.flatten end diff --git a/lib/puppet/provider/elb_loadbalancer/v2.rb b/lib/puppet/provider/elb_loadbalancer/v2.rb index a77e4196..2c6c57ac 100644 --- a/lib/puppet/provider/elb_loadbalancer/v2.rb +++ b/lib/puppet/provider/elb_loadbalancer/v2.rb @@ -7,14 +7,18 @@ def self.instances regions.collect do |region| - load_balancers = [] - region_client = elb_client(region) - region_client.describe_load_balancers.each do |response| - response.data.load_balancer_descriptions.collect do |lb| - load_balancers << new(load_balancer_to_hash(region, lb)) + begin + load_balancers = [] + region_client = elb_client(region) + region_client.describe_load_balancers.each do |response| + response.data.load_balancer_descriptions.collect do |lb| + load_balancers << new(load_balancer_to_hash(region, lb)) + end end + load_balancers + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - load_balancers end.flatten end diff --git a/lib/puppet/provider/route53_record.rb b/lib/puppet/provider/route53_record.rb index ba57621f..a7a14444 100644 --- a/lib/puppet/provider/route53_record.rb +++ b/lib/puppet/provider/route53_record.rb @@ -3,21 +3,25 @@ class Puppet::Provider::Route53Record < PuppetX::Puppetlabs::Aws def self.instances - zones_response = route53_client.list_hosted_zones() - records = [] - zones_response.data.hosted_zones.each do |zone| - records_response = route53_client.list_resource_record_sets(hosted_zone_id: zone.id) - records_response.data.resource_record_sets.each do |record| - records << new({ - name: record.name, - ensure: :present, - zone: zone.name, - ttl: record.ttl, - values: record.resource_records.map(&:value), - }) if record.type == record_type + begin + zones_response = route53_client.list_hosted_zones() + records = [] + zones_response.data.hosted_zones.each do |zone| + records_response = route53_client.list_resource_record_sets(hosted_zone_id: zone.id) + records_response.data.resource_record_sets.each do |record| + records << new({ + name: record.name, + ensure: :present, + zone: zone.name, + ttl: record.ttl, + values: record.resource_records.map(&:value), + }) if record.type == record_type + end end + records + rescue StandardError => e + raise PuppetX::Puppetlabs::FetchingAWSDataError.new(region, self.resource_type.name.to_s, e.message) end - records end def self.prefetch(resources) diff --git a/lib/puppet_x/puppetlabs/aws.rb b/lib/puppet_x/puppetlabs/aws.rb index b6e4b48a..1971aec9 100644 --- a/lib/puppet_x/puppetlabs/aws.rb +++ b/lib/puppet_x/puppetlabs/aws.rb @@ -1,5 +1,27 @@ module PuppetX module Puppetlabs + # We purposefully inherit from Exception here due to PUP-3656 + # If we throw something based on StandardError prior to Puppet 4 + # the exception will prevent the prefetch, but the provider will + # continue to run with incorrect data. + class FetchingAWSDataError < Exception + def initialize(region, type, message=nil) + @message = message + @region = region + @type = type + end + + def to_s + """Puppet detected a problem with the information returned from AWS +when looking up #{@type} in #{@region}. The specific error was: + +#{@message} + +Rather than report on #{@type} resources in an inconsistent state we have exited. +This could be because some other process is modifying AWS at the same time.""" + end + end + class Aws < Puppet::Provider def self.regions if ENV['AWS_REGION'] and not ENV['AWS_REGION'].empty?