Merge pull request #9466 from DFE-Digital/dd/move-migrations-to-worke…

…r-jobs-to-tackle-each-data-export-in-isolation

Offload data exports file migrations to workers

app/services/data_migrations/migrate_data_export_data_to_file.rb

@@ -4,8 +4,10 @@ class MigrateDataExportDataToFile
     MANUAL_RUN = true
 
     def change
-      DataExport.find_each do |data_export|
-        data_export.file.attach(io: CSV.new(data_export.data).to_io, filename: data_export.filename)
+      BatchDelivery.new(relation: DataExport.all, stagger_over: 4.hours, batch_size: 10).each do |next_batch_time, data_exports|
+        data_exports.each do |data_export|
+          DataExportFileMigrationWorker.perform_at(next_batch_time, data_export.id)
+        end
       end
     end
   end

app/workers/data_export_file_migration_worker.rb

@@ -0,0 +1,9 @@
+class DataExportFileMigrationWorker
+  include Sidekiq::Worker
+
+  def perform(data_export_id)
+    data_export = DataExport.find(data_export_id)
+
+    data_export.file.attach(io: CSV.new(data_export.data).to_io, filename: data_export.filename)
+  end
+end

spec/services/data_migrations/migrate_data_export_data_to_file_spec.rb

@@ -4,13 +4,8 @@
   let(:csv_string) { "email_template,send_count\nsign_in_email,1" }
 
   it "updates all existing DataExport records 'data' fields and puts them into 'file' attachments" do
-    data_export = create(:data_export, data: csv_string, file: nil)
+    create(:data_export, data: csv_string, file: nil)
 
-    described_class.new.change
-
-    data_export.reload
-
-    expect(data_export.file).to be_attached
-    expect(data_export.file.download).to eq(csv_string)
+    expect { described_class.new.change }.to change(DataExportFileMigrationWorker.jobs, :size).by(1)
   end
 end