Run as an auto-resuming background job (#1230)

Auto-resume whilst resetting PKs and capture progress/validity

app/jobs/data_migration_job.rb

@@ -0,0 +1,6 @@
+class DataMigrationJob < ApplicationJob
+  def run
+    require 'migrate_training'
+    MigrateTraining.new.call
+  end
+end

config/initializers/que.rb

@@ -7,6 +7,7 @@
     CompleteRegistrationMailJob: { cron: Rails.application.config.mail_job_interval },
     StartTrainingMailJob: { cron: Rails.application.config.mail_job_interval },
     ContinueTrainingMailJob: { cron: Rails.application.config.mail_job_interval },
+    DataMigrationJob: { cron: '0 18 * * *' },
   }
 end
 

db/migrate/20240616092250_add_state_to_user_answer.rb

@@ -0,0 +1,5 @@
+class AddStateToUserAnswer < ActiveRecord::Migration[7.1]
+  def change
+    add_column :user_answers, :state, :string
+  end
+end

lib/migrate_training.rb

@@ -8,7 +8,7 @@
 #   Assessment is created with the first Response meaning:
 #     - every Assessment has a minimum of 1 Response
 #     - every Assessment has a maximum score of 100%
-#
+#     - Assessments will outnumber UserAssessments
 #
 class MigrateTraining
   class Error < StandardError
@@ -22,8 +22,6 @@ class Error < StandardError
   option :verbose, Types::Bool, default: proc { true }, reader: :private
   # @return [Integer]
   option :batch, Types::Integer, default: proc { 100 }, reader: :private
-  # @return [Integer, nil]
-  option :resume, Types::Integer, optional: true, reader: :private
 
   # @return [nil]
   def call
@@ -38,19 +36,22 @@ def call
   # @return [PG::Result]
   def truncate!
     log 'Truncate responses and assessments'
+    UserAnswer.update_all(state: nil)
     ActiveRecord::Base.connection.execute 'TRUNCATE responses, assessments RESTART IDENTITY'
   end
 
   # @return [nil]
   def migrate!
-    UserAnswer.find_each(start: resume, batch_size: batch) do |user_answer|
+    UserAnswer.where(state: nil).find_each(batch_size: batch) do |user_answer|
       if valid?(user_answer)
         ActiveRecord::Base.transaction do
           response = process_user_answer(user_answer)
+          user_answer.update(state: 'done')
           log response.attributes.to_json, alert: false
         end
       else
-        log "User: #{user_answer.user_id} UserAnswer: #{user_answer.id}"
+        user_answer.update(state: 'skip')
+        log "Invalid UserAnswer: #{user_answer.id}", alert: false
       end
     end
   end
@@ -84,22 +85,30 @@ def process_user_assessment(user_answer)
     return unless user_answer.question.summative_question?
 
     if user_answer.user_assessment_id.nil?
-      assessment =
-        Assessment
-          .create_with(started_at: user_answer.created_at)
-          .find_or_create_by(user_id: user_answer.user_id, training_module: user_answer.module)
+      assessment_in_progress(user_answer)
     else
-      user_assessment = UserAssessment.find(user_answer.user_assessment_id)
-      params = assessment_params(user_assessment)
-      assessment = Assessment.find_or_create_by(params)
-
-      if assessment.score.nil?
-        if user_assessment.score.to_i > 100
-          assessment.update(score: 100)
-        else
-          assessment.update(score: user_assessment.score)
-        end
-      end
+      assessment_completed(user_answer)
+    end
+  end
+
+  # @param user_answer [UserAnswer]
+  # @return [Assessment]
+  def assessment_in_progress(user_answer)
+    Assessment
+      .create_with(started_at: user_answer.created_at)
+      .find_or_create_by(user_id: user_answer.user_id, training_module: user_answer.module)
+  end
+
+  # @param user_answer [UserAnswer]
+  # @return [Assessment]
+  def assessment_completed(user_answer)
+    user_assessment = UserAssessment.find(user_answer.user_assessment_id)
+    params = assessment_params(user_assessment)
+    assessment = Assessment.find_or_create_by(params)
+
+    if assessment.score.nil?
+      score = user_assessment.score.to_i > 100 ? 100 : user_assessment.score
+      assessment.update(score: score)
     end
 
     assessment
@@ -121,9 +130,7 @@ def assessment_params(user_assessment)
   # @return [Hash<Symbol=>Mixed>]
   def response_params(user_answer)
     {
-      id: user_answer.id,                                 # int db primary key
       user_id: user_answer.user_id,                       # int db foreign key
-      assessment_id: user_answer.user_assessment_id,      # int db foreign key
       training_module: user_answer.module,                # string cms key
       question_name: user_answer.name,                    # string cms key
       question_type: user_answer.question.question_type,  # string cms filter

lib/tasks/eyfs.rake

@@ -95,10 +95,4 @@ namespace :eyfs do
       end
     end
   end
-
-  desc 'Migrate training data to response and assessment models'
-  task migrate_training_data: :environment do
-    require 'migrate_training'
-    MigrateTraining.new.call
-  end
 end

spec/lib/migrate_training_spec.rb

@@ -23,6 +23,24 @@
       expect(Response.count).to eq 0
       expect(User.count).to eq 1
     end
+
+    it 'resets primary keys' do
+      create :assessment, user: user
+      create :response, user: user
+      expect(Assessment.last.id).to eq 1
+      expect(Response.last.id).to eq 1
+    end
+  end
+
+  context 'when data is invalid' do
+    before do
+      create :user_answer, user_id: user.id, name: 'foo', questionnaire_id: 0
+    end
+
+    it 'is ignored' do
+      expect { described_class.new.call }.to output(/Invalid UserAnswer/).to_stdout
+      expect(Response.count).to eq 0
+    end
   end
 
   context 'when data is valid' do
@@ -141,19 +159,10 @@
       # last question
       expect(bravo_assessment.completed_at).to be_nil
     end
-
-    it 'preserves response primary key' do
-      operation.call
-      user_answer = UserAnswer.find_by(name: '1-3-2-1')
-      response = Response.find_by(question_name: '1-3-2-1')
-
-      expect(response.id).to eq user_answer.id
-    end
   end
 
-  context 'when assessment score is invalid' do
+  context 'when assessment score exceeds 100' do
     before do
-      # Charlie (inconsistent)
       charlie_assessment =
         create :user_assessment,
                user_id: user.id,
@@ -175,20 +184,12 @@
     end
 
     it 'is corrected' do
-      expect(UserAssessment.count).to eq 1
       expect(Assessment.count).to be 1
-
-      expect(UserAnswer.count).to be 20
-      expect(Response.count).to be 20
-
-      expect(UserAssessment.first.user_answers.count).to eq 20
-      expect(Assessment.first.responses.count).to eq 20
-
       expect(Assessment.first.score).to eq 100.0
     end
   end
 
-  context 'with resume' do
+  context 'when resuming' do
     before do
       assessment_1 =
         create :user_assessment,
@@ -208,8 +209,17 @@
     end
 
     it 'migrates remaining data' do
-      last_user_answer_id = UserAnswer.last.id
+      # in progress
+      create_list :user_answer, 3,
+                  user_id: user.id,
+                  name: '1-3-2-1',
+                  module: 'bravo',
+                  assessments_type: 'summative_assessment',
+                  user_assessment_id: nil,
+                  questionnaire_id: 0,
+                  created_at: Time.zone.now
 
+      # completed
       assessment_2 =
         create :user_assessment,
                user_id: user.id,
@@ -224,16 +234,16 @@
                   questionnaire_id: 0,
                   created_at: Time.zone.now
 
-      described_class.new(verbose: false, resume: last_user_answer_id + 1).call
-
-      expect(UserAnswer.pluck(:id)[9]).to eq last_user_answer_id
-      expect(UserAnswer.pluck(:id)[19]).to eq UserAnswer.last.id
+      described_class.new(verbose: false).call
 
       expect(UserAssessment.count).to eq 2
-      expect(Assessment.count).to be 2
+      expect(Assessment.count).to be 3
+
+      expect(UserAnswer.count).to be 23
+      expect(Response.count).to be 23
 
-      expect(UserAnswer.count).to be 20
-      expect(Response.count).to be 20
+      expect(Assessment.all.map(&:training_module)).to eq %w[alpha bravo charlie]
+      expect(Assessment.all.map(&:score)).to eq [0.0, nil, 0.0]
     end
   end
 end