Attempt to atomically update repository metadata

Author: hpvb

lib/yum_s3_sync/repo_syncer.rb

@@ -20,7 +20,7 @@ def sync
       s3_downloader = YumS3Sync::S3Downloader.new(@target_bucket, @target_base)
       dest_repository = YumS3Sync::YumRepository.new(s3_downloader)
 
-      s3_uploader = YumS3Sync::S3Uploader.new(@target_bucket, @target_base, http_downloader, @dry_run)
+      s3_uploader = YumS3Sync::S3Uploader.new(@target_bucket, @target_base, @dry_run)
 
       s3_file_lister = YumS3Sync::S3FileLister.new(@target_bucket, @target_base)
       s3_deleter = YumS3Sync::S3Deleter.new(@target_bucket, @target_base, @dry_run)
@@ -33,17 +33,19 @@ def sync
         if !dest_repository.metadata[type] || dest_repository.metadata[type][:checksum] != file[:checksum]
           new_metadata = true
         end
-          
+
         metadata.push file[:href]
       end
 
       new_packages.each do |package|
-        s3_uploader.upload(package, @keep)
+        if @keep && ! s3_uploader.file_exists?(package)
+          s3_uploader.upload(package, http_downloader.download(package))
+        end
       end
 
       if !dest_repository.exists? || !new_packages.empty? || new_metadata
-        metadata.each do |file|
-          s3_uploader.upload(file, true)
+        source_repository.metadata.each do |type, file|
+          s3_uploader.upload(file[:href], file[:file])
         end
       end
 
@@ -59,7 +61,7 @@ def sync
       puts "Locating missing files"
       source_repository.packages.each do |package, data|
         unless file_names.include? package
-          s3_uploader.upload(package, true)
+          s3_uploader.upload(package, http_downloader.download(package))
         end
       end
 

lib/yum_s3_sync/s3_uploader.rb

@@ -2,35 +2,39 @@
 
 module YumS3Sync
   class S3Uploader
-    def initialize(bucket, prefix, downloader, dry_run = false)
+    def initialize(bucket, prefix, dry_run = false)
       @bucket = bucket
       @prefix = prefix
-      @downloader = downloader
       @dry_run = dry_run
+      @s3 = AWS::S3.new
     end
 
-    def upload(file, overwrite = false)
-      retries = 0
-      s3 = AWS::S3.new
-
-      begin
-        target = "#{@prefix}/#{file}"
+    def file_exists?(source_url)
+        target = "#{@prefix}/#{source_url}"
         target.gsub!(/\/+/, '/')
-        dest_obj = s3.buckets[@bucket].objects[target]
+        dest_obj = @s3.buckets[@bucket].objects[target]
 
-        if dest_obj.exists? && ! overwrite
-          puts "Already exists: skipping #{@bucket}::#{target}" 
-          return
+        if dest_obj.exists? 
+          return true
         end
 
-        source_file = @downloader.download(file)
+        return false
+    end
+
+    def upload(source_url, file)
+      retries = 0
+
+      begin
+        target = "#{@prefix}/#{source_url}"
+        target.gsub!(/\/+/, '/')
+        dest_obj = @s3.buckets[@bucket].objects[target]
 
         if @dry_run 
           puts "Dry-run: Uploading #{@bucket}::#{target}"
         else
           puts "Uploading #{@bucket}::#{target}"
           dest_obj.delete if dest_obj.exists?
-          dest_obj.write(:file => source_file)
+          dest_obj.write(:file => file)
         end
       rescue StandardError => e
         if retries < 10

lib/yum_s3_sync/version.rb

@@ -1,3 +1,3 @@
 module YumS3Sync
-  VERSION = '0.0.13'
+  VERSION = '0.0.14'
 end

lib/yum_s3_sync/yum_repository.rb

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'zlib'
+require 'stringio'
 require 'nokogiri'
 require 'rexml/document'
 require 'rexml/streamlistener'
@@ -15,15 +16,23 @@ def initialize(downloader)
 
       repomd_file = @downloader.download('repodata/repomd.xml')
       if repomd_file
-        doc = Nokogiri::XML(repomd_file)
+        repomd = StringIO.new(repomd_file.read())
+
+        doc = Nokogiri::XML(repomd)
         doc.xpath("//xmlns:data").each do |file|
-          metadata[file['type']] = {
-            :href => file.xpath('xmlns:location')[0]['href'],
-            :checksum => file.xpath('xmlns:checksum')[0].child.to_s
-          }
+          href = file.xpath('xmlns:location')[0]['href']
+          f = @downloader.download(href)
+          if f
+            metadata[file['type']] = {
+              :href => href,
+              :checksum => file.xpath('xmlns:checksum')[0].child.to_s,
+              :file => StringIO.new(f.read())
+            }
+          end
         end
 
-        @metadata['repomd'] = { :href => 'repodata/repomd.xml' }
+        repomd.rewind()
+        @metadata['repomd'] = { :href => 'repodata/repomd.xml', :file => repomd }
       else
         @metadata = { 'primary' => nil }
       end
@@ -32,18 +41,16 @@ def initialize(downloader)
     def parse_packages
       return {} unless @metadata['primary']
 
-      primary_file = @downloader.download(@metadata['primary'][:href])
-      return {} unless primary_file
-
       puts "Parsing #{@metadata['primary'][:href]}"
-      gzstream = Zlib::GzipReader.new(primary_file)
+      gzstream = Zlib::GzipReader.new(@metadata['primary'][:file])
 
       doc = Nokogiri::XML(gzstream)
       packages = {}
       doc.xpath("//xmlns:package").each do |package|
         packages[package.xpath("xmlns:location")[0]['href']] = package.xpath("xmlns:checksum")[0].child.to_s
       end
 
+      @metadata['primary'][:file].rewind()
       packages
     end