From 95c64e255589c62f677430266aa0a8ea3abb9b87 Mon Sep 17 00:00:00 2001 From: Jaymz Julian Date: Tue, 21 Aug 2018 14:32:19 +1000 Subject: [PATCH 1/3] aws v3 support Signed-off-by: Jaymz Julian --- .travis.yml | 6 +++--- chef-provisioning-aws.gemspec | 17 ++++++++++++++++- lib/chef/provider/aws_key_pair.rb | 1 - lib/chef/provider/aws_nat_gateway.rb | 2 +- .../provisioning/aws_driver/aws_resource.rb | 16 +++++++++++++++- .../provisioning/aws_driver/credentials2.rb | 1 - lib/chef/provisioning/aws_driver/driver.rb | 18 +++++++++++++++--- lib/chef/provisioning/aws_driver/version.rb | 2 +- lib/chef/resource/aws_nat_gateway.rb | 2 +- spec/aws_support.rb | 16 +++++++++++++++- spec/integration/aws_rds_subnet_group_spec.rb | 2 +- tools/purge_zone.rb | 2 +- 12 files changed, 69 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 913f4ea0..91cd291b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,10 @@ env: - RAKE_TASK=spec # We run the tests in different environments so they don't try to overwrite each other (deleting, creating the same VPC) # If we need to we can get rid of this but its nice for speed - - RAKE_TASK="travis[integration]" AWS_TRAVIS_DRIVER=aws::us-west-2 - - RAKE_TASK="travis[super_slow]" AWS_TRAVIS_DRIVER=aws::us-west-1 + - RAKE_TASK="travis[integration]" AWS_TEST_DRIVER=aws::us-west-2 + - RAKE_TASK="travis[super_slow]" AWS_TEST_DRIVER=aws::us-west-1 # machine_image is a special snowflake - they take so long to run we need to give them their own builder - - RAKE_TASK="machine_image" AWS_TRAVIS_DRIVER=aws::eu-west-1 + - RAKE_TASK="machine_image" AWS_TEST_DRIVER=aws::eu-west-1 bundler_args: --jobs 7 --without docs debug diff --git a/chef-provisioning-aws.gemspec b/chef-provisioning-aws.gemspec index 3ee43d07..4c8ac1cb 100644 --- a/chef-provisioning-aws.gemspec +++ b/chef-provisioning-aws.gemspec @@ -17,7 +17,22 @@ Gem::Specification.new do |s| s.add_dependency "chef-provisioning", ">= 1.0", "< 3.0" - s.add_dependency "aws-sdk", [">= 2.2.18", "< 3.0"] + # all currently supported AWS services + s.add_dependency "aws-sdk-core", [">= 3.0", "< 4.0"] + s.add_dependency "aws-sdk-ec2", [">= 1.42.0", "< 2.0"] + s.add_dependency "aws-sdk-s3", [">= 1.17.0", "< 2.0"] + s.add_dependency "aws-sdk-rds", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-route53", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-autoscaling", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-cloudwatch", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-cloudsearch", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-cloudsearchdomain", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-elasticache", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-iam", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-opsworks", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-sns", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-sqs", [">= 1.0", "< 4.0"] + s.add_dependency "retryable", "~> 2.0", ">= 2.0.1" s.add_dependency "ubuntu_ami", "~> 0.4", ">= 0.4.1" diff --git a/lib/chef/provider/aws_key_pair.rb b/lib/chef/provider/aws_key_pair.rb index 15856d32..82e2cc24 100644 --- a/lib/chef/provider/aws_key_pair.rb +++ b/lib/chef/provider/aws_key_pair.rb @@ -1,6 +1,5 @@ require "chef/provider/lwrp_base" require "chef/provisioning/aws_driver/aws_provider" -require "aws-sdk" class Chef::Provider::AwsKeyPair < Chef::Provisioning::AWSDriver::AWSProvider provides :aws_key_pair diff --git a/lib/chef/provider/aws_nat_gateway.rb b/lib/chef/provider/aws_nat_gateway.rb index af5fadfd..9081d50b 100644 --- a/lib/chef/provider/aws_nat_gateway.rb +++ b/lib/chef/provider/aws_nat_gateway.rb @@ -1,4 +1,4 @@ -# require 'chef/provisioning/aws_driver/aws_provider' +# require "chef/provisioning/aws_driver/aws_provider" require "retryable" class Chef::Provider::AwsNatGateway < Chef::Provisioning::AWSDriver::AWSProvider diff --git a/lib/chef/provisioning/aws_driver/aws_resource.rb b/lib/chef/provisioning/aws_driver/aws_resource.rb index 8b24d335..f082b008 100644 --- a/lib/chef/provisioning/aws_driver/aws_resource.rb +++ b/lib/chef/provisioning/aws_driver/aws_resource.rb @@ -1,4 +1,18 @@ -require "aws-sdk" +# Parts of AWS we support - for now this is scattershot, later this will get cleaned up +require "aws-sdk-s3" +require "aws-sdk-ec2" +require "aws-sdk-rds" +require "aws-sdk-route53" +require "aws-sdk-autoscaling" +require "aws-sdk-cloudwatch" +require "aws-sdk-cloudsearch" +require "aws-sdk-cloudsearchdomain" +require "aws-sdk-elasticache" +require "aws-sdk-iam" +require "aws-sdk-opsworks" +require "aws-sdk-sns" +require "aws-sdk-sqs" + require "chef/provisioning/aws_driver/super_lwrp" require "chef/provisioning/chef_managed_entry_store" # Enough resources will eventually require this that we put 1 require in here diff --git a/lib/chef/provisioning/aws_driver/credentials2.rb b/lib/chef/provisioning/aws_driver/credentials2.rb index ad5da639..bf80cee3 100644 --- a/lib/chef/provisioning/aws_driver/credentials2.rb +++ b/lib/chef/provisioning/aws_driver/credentials2.rb @@ -1,4 +1,3 @@ -require "aws-sdk" require "aws-sdk-core/credentials" require "aws-sdk-core/shared_credentials" require "aws-sdk-core/instance_profile_credentials" diff --git a/lib/chef/provisioning/aws_driver/driver.rb b/lib/chef/provisioning/aws_driver/driver.rb index 75895f3b..1c93bf95 100644 --- a/lib/chef/provisioning/aws_driver/driver.rb +++ b/lib/chef/provisioning/aws_driver/driver.rb @@ -20,13 +20,26 @@ require "chef/provisioning/aws_driver/aws_tagger" require "yaml" -require "aws-sdk" require "retryable" require "ubuntu_ami" require "base64" +# Parts of AWS we support - for now this is scattershot, later this will get cleaned up +require "aws-sdk-s3" +require "aws-sdk-ec2" +require "aws-sdk-rds" +require "aws-sdk-route53" +require "aws-sdk-autoscaling" +require "aws-sdk-cloudwatch" +require "aws-sdk-cloudsearch" +require "aws-sdk-cloudsearchdomain" +require "aws-sdk-elasticache" +require "aws-sdk-iam" +require "aws-sdk-opsworks" +require "aws-sdk-sns" +require "aws-sdk-sqs" + # loads the entire aws-sdk -Aws.eager_autoload! AWS_V2_SERVICES = { "EC2" => "ec2", "Route53" => "route53", @@ -38,7 +51,6 @@ "CloudWatch" => "cloudwatch", "AutoScaling" => "auto_scaling" }.freeze -Aws.eager_autoload!(services: AWS_V2_SERVICES.keys) # Need to load the resources after the SDK because `aws_sdk_types` can mess # up AWS loading if they are loaded too early diff --git a/lib/chef/provisioning/aws_driver/version.rb b/lib/chef/provisioning/aws_driver/version.rb index 473a62e5..d8e0a3f0 100644 --- a/lib/chef/provisioning/aws_driver/version.rb +++ b/lib/chef/provisioning/aws_driver/version.rb @@ -1,7 +1,7 @@ class Chef module Provisioning module AWSDriver - VERSION = "3.0.5".freeze + VERSION = "4.0.0" end end end diff --git a/lib/chef/resource/aws_nat_gateway.rb b/lib/chef/resource/aws_nat_gateway.rb index 5c37e7c1..fbbbd43e 100644 --- a/lib/chef/resource/aws_nat_gateway.rb +++ b/lib/chef/resource/aws_nat_gateway.rb @@ -15,7 +15,7 @@ # We provide this class because the AWS SDK V2 does not provide it (as of # May 2016). We copied the pattern in their SDK so when they do add a real # resource there shouldn't be a need for much translation. -class Aws::EC2::NatGateway < ::Aws::Resources::Resource +class Aws::EC2::NatGateway # < ::Aws::Resources::Resource attr_reader :resource, :id, :nat_gateway_id, :vpc_id, :subnet_id, :nat_gateway_addresses def initialize(id, options = {}) diff --git a/spec/aws_support.rb b/spec/aws_support.rb index 394f5e10..8ff112f5 100644 --- a/spec/aws_support.rb +++ b/spec/aws_support.rb @@ -20,7 +20,21 @@ def self.extended(other) require "aws_support/aws_resource_run_wrapper" # Add AWS to the list of objects which can be matched against a Hash or Array - require "aws-sdk" + # Parts of AWS we support - for now this is scattershot, later this will get cleaned up + require "aws-sdk-s3" + require "aws-sdk-ec2" + require "aws-sdk-rds" + require "aws-sdk-route53" + require "aws-sdk-autoscaling" + require "aws-sdk-cloudwatch" + require "aws-sdk-cloudsearch" + require "aws-sdk-cloudsearchdomain" + require "aws-sdk-elasticache" + require "aws-sdk-iam" + require "aws-sdk-opsworks" + require "aws-sdk-sns" + require "aws-sdk-sqs" + require "aws_support/deep_matcher/matchable_object" require "aws_support/deep_matcher/matchable_array" DeepMatcher::MatchableObject.matchable_classes << proc { |o| o.class.name =~ /^(AWS|Aws)::(AutoScaling|EC2|ELB|IAM|S3|RDS|CloudSearch|CloudWatch|Route53|ElasticsearchService)($|::)/ } diff --git a/spec/integration/aws_rds_subnet_group_spec.rb b/spec/integration/aws_rds_subnet_group_spec.rb index d2896f2d..97ea18c9 100644 --- a/spec/integration/aws_rds_subnet_group_spec.rb +++ b/spec/integration/aws_rds_subnet_group_spec.rb @@ -1,5 +1,5 @@ require "spec_helper" -require "aws-sdk" +require "aws-sdk-rds" require "set" describe Chef::Resource::AwsRdsSubnetGroup do diff --git a/tools/purge_zone.rb b/tools/purge_zone.rb index 7a534c4b..be8aa8ef 100755 --- a/tools/purge_zone.rb +++ b/tools/purge_zone.rb @@ -2,7 +2,7 @@ require "json" require "pp" -require "aws-sdk" +require "aws-sdk-route53" # it happens that you end up with a HostedZone with no Chef Server entry, and destroying it manually is # tedious, because you have to delete all the ResourceRecordSets. this script will handle that for you. From f0e766be6a71c07a7373ba4ddf0441e7fd260b4c Mon Sep 17 00:00:00 2001 From: Jaymz Julian Date: Mon, 1 Oct 2018 16:22:32 +1000 Subject: [PATCH 2/3] fixup AWS gem deps to be ~> 1.0 as per code review Signed-off-by: Jaymz Julian --- chef-provisioning-aws.gemspec | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/chef-provisioning-aws.gemspec b/chef-provisioning-aws.gemspec index 4c8ac1cb..9b10568a 100644 --- a/chef-provisioning-aws.gemspec +++ b/chef-provisioning-aws.gemspec @@ -19,19 +19,19 @@ Gem::Specification.new do |s| # all currently supported AWS services s.add_dependency "aws-sdk-core", [">= 3.0", "< 4.0"] - s.add_dependency "aws-sdk-ec2", [">= 1.42.0", "< 2.0"] - s.add_dependency "aws-sdk-s3", [">= 1.17.0", "< 2.0"] - s.add_dependency "aws-sdk-rds", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-route53", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-autoscaling", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-cloudwatch", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-cloudsearch", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-cloudsearchdomain", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-elasticache", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-iam", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-opsworks", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-sns", [">= 1.0", "< 4.0"] - s.add_dependency "aws-sdk-sqs", [">= 1.0", "< 4.0"] + s.add_dependency "aws-sdk-ec2", ["~> 1.0" ] + s.add_dependency "aws-sdk-s3", ["~> 1.0" ] + s.add_dependency "aws-sdk-rds", ["~> 1.0" ] + s.add_dependency "aws-sdk-route53", ["~> 1.0" ] + s.add_dependency "aws-sdk-autoscaling", ["~> 1.0" ] + s.add_dependency "aws-sdk-cloudwatch", ["~> 1.0" ] + s.add_dependency "aws-sdk-cloudsearch", ["~> 1.0" ] + s.add_dependency "aws-sdk-cloudsearchdomain", ["~> 1.0" ] + s.add_dependency "aws-sdk-elasticache", ["~> 1.0" ] + s.add_dependency "aws-sdk-iam", ["~> 1.0" ] + s.add_dependency "aws-sdk-opsworks", ["~> 1.0" ] + s.add_dependency "aws-sdk-sns", ["~> 1.0" ] + s.add_dependency "aws-sdk-sqs", ["~> 1.0" ] s.add_dependency "retryable", "~> 2.0", ">= 2.0.1" s.add_dependency "ubuntu_ami", "~> 0.4", ">= 0.4.1" From e43b6196fe78484f1bb61abe9418ed2880e136c9 Mon Sep 17 00:00:00 2001 From: Jaymz Julian Date: Mon, 1 Oct 2018 16:25:48 +1000 Subject: [PATCH 3/3] add a missing require to aws_key_pair Signed-off-by: Jaymz Julian --- lib/chef/provider/aws_key_pair.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/chef/provider/aws_key_pair.rb b/lib/chef/provider/aws_key_pair.rb index 82e2cc24..03b0e324 100644 --- a/lib/chef/provider/aws_key_pair.rb +++ b/lib/chef/provider/aws_key_pair.rb @@ -1,5 +1,6 @@ require "chef/provider/lwrp_base" require "chef/provisioning/aws_driver/aws_provider" +require "aws-sdk-ec2" class Chef::Provider::AwsKeyPair < Chef::Provisioning::AWSDriver::AWSProvider provides :aws_key_pair