Add parameter to remove deleted packages from bucket

lib/deb/s3/cli.rb

@@ -179,7 +179,7 @@ def upload(*files)
       release  = Deb::S3::Release.retrieve(options[:codename], options[:origin], options[:suite], options[:cache_control])
       manifests = {}
       release.architectures.each do |arch|
-        manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
+        manifests[arch] = Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload], options[:do_package_remove])
       end
 
       packages_arch_all = []
@@ -217,7 +217,7 @@ def upload(*files)
         end
 
         # retrieve the manifest for the arch if we don't have it already
-        manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload])
+        manifests[arch] ||= Deb::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload], options[:do_package_remove])
 
         # add package in manifests
         begin
@@ -287,7 +287,7 @@ def list
     rows = archs.map { |arch|
       manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
                                             arch, options[:cache_control],
-                                            false, false)
+                                            false, false, false)
       manifest.packages.map do |package|
         if options[:long]
           package.generate(options[:codename])
@@ -416,12 +416,12 @@ def copy(package_name, to_codename, to_component)
       from_manifest = Deb::S3::Manifest.retrieve(options[:codename],
                                                  component, arch,
                                                  options[:cache_control],
-                                                 false, options[:skip_package_upload])
+                                                 false, options[:skip_package_upload], options[:do_package_remove])
       to_release = Deb::S3::Release.retrieve(to_codename)
       to_manifest = Deb::S3::Manifest.retrieve(to_codename, to_component, arch,
                                                options[:cache_control],
                                                options[:fail_if_exists],
-                                               options[:skip_package_upload])
+                                               options[:skip_package_upload], options[:do_package_remove])
       packages = from_manifest.packages.select { |p|
         p.name == package_name &&
           (versions.nil? || versions.include?(p.full_version))
@@ -479,6 +479,11 @@ def copy(package_name, to_codename, to_component)
     "specified, ALL VERSIONS will be deleted. Fair warning. " +
     "E.g. --versions \"0.1 0.2 0.3\""
 
+  option :do_package_remove,
+    :default  => false,
+    :type     => :boolean,
+    :desc     => "Whether to remove packages from the bucket when deleting them from the manifest."
+
   def delete(package)
     if package.nil?
       error("You must specify a package name.")
@@ -516,7 +521,7 @@ def delete(package)
       end
       all_found = 0
       selected_arch.each { |ar|
-        manifest = Deb::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false, options[:skip_package_upload])
+        manifest = Deb::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false, options[:skip_package_upload], options[:do_package_remove])
 
         deleted = manifest.delete_package(package, versions)
         all_found += deleted.length
@@ -575,7 +580,7 @@ def verify
       log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
       manifest = Deb::S3::Manifest.retrieve(options[:codename], component,
                                             arch, options[:cache_control], false,
-                                            options[:skip_package_upload])
+                                            options[:skip_package_upload], options[:do_package_remove])
       missing_packages = []
 
       manifest.packages.each do |p|

lib/deb/s3/manifest.rb

@@ -13,25 +13,29 @@ class Deb::S3::Manifest
   attr_accessor :architecture
   attr_accessor :fail_if_exists
   attr_accessor :skip_package_upload
+  attr_accessor :do_package_remove
 
   attr_accessor :files
 
   attr_reader :packages
   attr_reader :packages_to_be_upload
+  attr_reader :packages_to_be_deleted
 
   def initialize
     @packages = []
     @packages_to_be_upload = []
+    @packages_to_be_deleted = []
     @component = nil
     @architecture = nil
     @files = {}
     @cache_control = ""
     @fail_if_exists = false
     @skip_package_upload = false
+    @do_package_remove = true # TODO(pbovbel) false by default
   end
 
   class << self
-    def retrieve(codename, component, architecture, cache_control, fail_if_exists, skip_package_upload=false)
+    def retrieve(codename, component, architecture, cache_control, fail_if_exists, skip_package_upload=false, do_package_remove=false)
       m = if s = Deb::S3::Utils.s3_read("dists/#{codename}/#{component}/binary-#{architecture}/Packages")
         self.parse_packages(s)
       else
@@ -44,6 +48,7 @@ def retrieve(codename, component, architecture, cache_control, fail_if_exists, s
       m.cache_control = cache_control
       m.fail_if_exists = fail_if_exists
       m.skip_package_upload = skip_package_upload
+      m.do_package_remove = do_package_remove
       m
     end
 
@@ -91,6 +96,7 @@ def delete_package(pkg, versions=nil)
         end
     }
     deleted = @packages - new_packages
+    packages_to_be_deleted.push(*deleted)
     @packages = new_packages
     deleted
   end
@@ -110,6 +116,14 @@ def write_to_s3
       end
     end
 
+    if self.do_package_remove
+      # delete any packages that need to be deleted
+      @packages_to_be_deleted.each do |pkg|
+        yield pkg.url_filename(@codename) if block_given?
+        s3_remove(pkg.url_filename(@codename))
+      end
+    end
+
     # generate the Packages file
     pkgs_temp = Tempfile.new("Packages")
     pkgs_temp.write manifest