Validate md5sum of downloaded s3 objects and retry on mismatch

Transient network errors, timeouts, Ruby Net::HTTP bugs, and other things can
cause s3 downloads to be incomplete or outright incorrect.
Only succeed a download if the md5 of the downloaded object
matches the md5 known to s3.

Author: Craig Hagan

libraries/s3_file.rb

@@ -128,22 +128,88 @@ def self.get_digests_from_headers(headers)
     return {"md5" => etag}.merge(digests)
   end
 
-  def self.get_digests_from_s3(bucket,url,path,aws_access_key_id,aws_secret_access_key,token, region)
-    response = do_request("HEAD", url, bucket, path, aws_access_key_id, aws_secret_access_key, token, region)
-    return self.get_digests_from_headers(response.headers)
+  def self.get_digests_from_s3(bucket,path,aws_access_key_id,aws_secret_access_key,token,timeout=300,open_timeout=10,retries=5)
+    now, auth_string = get_s3_auth("HEAD", bucket,path,aws_access_key_id,aws_secret_access_key, token)
+    max_tries = retries + 1
+    headers = build_headers(now, auth_string, token)
+    saved_exception = nil
+
+    while (max_tries > 0)
+      begin
+
+        response = RestClient.head('https://%s.s3.amazonaws.com%s' % [bucket,path], headers)
+
+        etag = response.headers[:etag].gsub('"','')
+        digest = response.headers[:x_amz_meta_digest]
+        digests = digest.nil? ? {} : Hash[digest.split(",").map {|a| a.split("=")}]
+
+        return {"md5" => etag}.merge(digests)
+
+        rescue => e
+           max_tries = max_tries - 1
+           saved_exception = e
+      end
+    end
+    raise saved_exception
+  end
+
+  def self.validate_download_checksum(response)
+    # Default to not checking md5 sum of downloaded objects
+    # per http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+    # If an object is created by either the Multipart Upload or Part Copy operation,
+    # the ETag is not an MD5 digest, regardless of the method of encryption
+    # however, if present, x-amz-meta-digest will contain the digest, so
+    # try if we see enough information and verify_md5 is set.
+    if response.headers[:x_amz_meta_digest]
+      return self.verify_md5_checksum(response.headers[:x_amz_meta_digest_md5].gsub('"',''), response.file.path)
+    else
+      server_side_encryption_customer_algorithm = response.headers[:x_amz_server_side_encryption_customer_algorithm]
+      server_side_encryption = response.headers[:x_amz_server_side_encryption]
+      if server_side_encryption_customer_algorithm.nil? and server_side_encryption != "aws:kms"
+        return self.verify_md5_checksum(response.headers[:etag].gsub('"',''), response.file.path)
+      else
+        # If we do not have the x-amz-meta-digest-md5 header, we
+        # cannot validate objects encrypted with SSE-C or SSE-KMS,
+        # because the ETag will not be the MD5 digest.  Assume it is
+        # valid in those cases.
+        return true
+      end
+    end
   end
 
-  def self.get_from_s3(bucket, url, path, aws_access_key_id, aws_secret_access_key, token, region = nil, verify_md5=true)
+
+  def self.get_from_s3(bucket, url, path, aws_access_key_id, aws_secret_access_key, token, verify_md5=false, region = nil)
     response = nil
     retries = 5
     for attempts in 0..retries
       begin
         response = do_request("GET", url, bucket, path, aws_access_key_id, aws_secret_access_key, token, region)
 
+        # check the length of the downloaded object,
+        # make sure we didn't get nailed by
+        # a quirk in Net::HTTP class from the Ruby standard library.
+        # Net::HTTP has the behavior (and I would call this a bug) that if the
+        # connection gets reset in the middle of transferring the response,
+        # it silently truncates the response back to the caller without throwing an exception.
+        # ** See https://github.com/ruby/ruby/blob/trunk/lib/net/http/response.rb#L291
+        # and https://github.com/ruby/ruby/blob/trunk/lib/net/protocol.rb#L99 .
+        # It attempts to read up to Content-Length worth of bytes, but if hits an early EOF,
+        # it just returns without throwing an exception (the ignore_eof flag).
+
+        length = response.headers[:content_length].to_i()
+        if not length.nil? and response.file.size() != length
+          raise "Downloaded object size (#{response.file.size()}) does not match expected content_length (#{length})"
+        end
+
+        # default to not checking md5 sum of downloaded objects
+        # per http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+        # If an object is created by either the Multipart Upload or Part Copy operation,
+        # the ETag is not an MD5 digest, regardless of the method of encryption
+        # however, if present, x-amz-meta-digest will contain the digest, so
+        # try if we see enough information and verify_md5 is set.
         if verify_md5
-          md5 = self.get_digests_from_headers(response.headers)["md5"]
-          if not self.verify_md5_checksum(md5,response.file.path)
-            raise "unable to validate md5 checksum of downloaded object"
+          if not self.validate_download_checksum(response)
+            raise "Downloaded object has an md5sum which differs from the expected value provided by S3"
           end
         end
 
@@ -170,6 +236,25 @@ def self.get_from_s3(bucket, url, path, aws_access_key_id, aws_secret_access_key
     end
   end
 
+  def self.get_s3_auth(method, bucket,path,aws_access_key_id,aws_secret_access_key, token)
+    now = Time.now().utc.strftime('%a, %d %b %Y %H:%M:%S GMT')
+    string_to_sign = "#{method}\n\n\n%s\n" % [now]
+
+    if token
+      string_to_sign += "x-amz-security-token:#{token}\n"
+    end
+
+    string_to_sign += "/%s%s" % [bucket,path]
+
+    digest = digest = OpenSSL::Digest::Digest.new('sha1')
+    signed = OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)
+    signed_base64 = Base64.encode64(signed)
+
+    auth_string = 'AWS %s:%s' % [aws_access_key_id,signed_base64]
+
+    [now,auth_string]
+  end
+
   def self.aes256_decrypt(key, file)
     Chef::Log.debug("Decrypting S3 file.")
     key = key.strip

providers/default.rb

@@ -62,7 +62,7 @@
   end
 
   if download
-    response = S3FileLib::get_from_s3(new_resource.bucket, new_resource.s3_url, remote_path, aws_access_key_id, aws_secret_access_key, token)
+    response = S3FileLib::get_from_s3(new_resource.bucket, new_resource.s3_url, remote_path, aws_access_key_id, aws_secret_access_key, token, new_resource.verify_md5)
 
     # not simply using the file resource here because we would have to buffer
     # whole file into memory in order to set content this solves

resources/default.rb

@@ -11,6 +11,7 @@
 attribute :mode, :kind_of => [String, Integer, NilClass], :default => nil
 attribute :decryption_key, :kind_of => String, :default => nil
 attribute :decrypted_file_checksum, :kind_of => String, :default => nil
+attribute :verify_md5, :kind_of => [TrueClass, FalseClass], :default => false
 
 def initialize(*args)
   super