From 909c4596fb95a9505743c6cb309ff0d7ffe543e9 Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Sun, 5 Jan 2025 22:57:45 -0800 Subject: [PATCH 1/8] Update recurring.yml --- config/recurring.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/recurring.yml b/config/recurring.yml index 6438588d22..4204dc8749 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -11,3 +11,7 @@ # a_cleanup_task: # command: "DeletedStuff.clear_all" # schedule: every day at 9am +production: + repopulate_location_and_observation_center_lat_lng: + command: "Location.update_box_area_and_center_columns" + schedule: every saturday at midnight From 0ca7bedef3d1ba53eb9f018819b3274f6a13cd11 Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Mon, 6 Jan 2025 15:00:00 -0800 Subject: [PATCH 2/8] Return counts from `update_box_area_and_center_columns` and log them in the job --- .../update_box_area_and_center_columns_job.rb | 26 +++++++++++++++++++ app/models/location.rb | 19 +++++++++----- ...te_box_area_and_center_columns_job_test.rb | 7 +++++ 3 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 app/jobs/update_box_area_and_center_columns_job.rb create mode 100644 test/jobs/update_box_area_and_center_columns_job_test.rb diff --git a/app/jobs/update_box_area_and_center_columns_job.rb b/app/jobs/update_box_area_and_center_columns_job.rb new file mode 100644 index 0000000000..f629a9cf42 --- /dev/null +++ b/app/jobs/update_box_area_and_center_columns_job.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class UpdateBoxAreaAndCenterColumnsJob < ApplicationJob + queue_as :default + + def perform(*args) + log("Starting UpdateBoxAreaAndCenterColumnsJob.perform") + # This should be zero, but count just in case. + loc_count = Location.where(box_area: nil).count + log("Found #{loc_count} locations without a box_area.") + # Count the observations associated with locations that are under the + # max area, but haven't got a center lat/lng, and log what we're updating. + obs_count = Observation.in_box_of_max_area.where(location_lat: nil).count + log("Found #{obs_count} observations where the associated location was " \ + "small enough, but the obs didn't have a location_lat/lng.") + return if args[:dry_run] + + # Do the update. This returns counts of locations/observations updated. + loc_updated, obs_centered, obs_center_nulled = + Location.update_box_area_and_center_columns + + log("Updated #{loc_updated} locations' box_area and center_lat/lng.") + log("Updated #{obs_centered} observations' location_lat/lng.") + log("Nulled #{obs_center_nulled} observations' location_lat/lng.") + end +end diff --git a/app/models/location.rb b/app/models/location.rb index a4f43c11fa..8cb65a40e5 100644 --- a/app/models/location.rb +++ b/app/models/location.rb @@ -311,17 +311,22 @@ def update_observation_center_columns # Can populate columns after migration, or be run as part of a recurring job. def self.update_box_area_and_center_columns # update the locations - update_all(update_center_and_area_sql) + loc_updated = update_all(update_center_and_area_sql) # give center points to associated observations in batches by location_id # Observation must be unscoped in order to join to locations. # (removing default_scope) - Observation.unscoped.in_box_of_max_area.group(:location_id).update_all( - location_lat: Location[:center_lat], location_lng: Location[:center_lng] - ) + obs_centered = Observation.unscoped. + in_box_of_max_area.group(:location_id).update_all( + location_lat: Location[:center_lat], + location_lng: Location[:center_lng] + ) # null center points where area is above the threshold - Observation.unscoped.in_box_gt_max_area.group(:location_id).update_all( - location_lat: nil, location_lng: nil - ) + obs_center_nulled = Observation.unscoped. + in_box_gt_max_area.group(:location_id).update_all( + location_lat: nil, location_lng: nil + ) + # Return counts + [loc_updated, obs_centered, obs_center_nulled] end # Let attached observations update their cache if these fields changed. diff --git a/test/jobs/update_box_area_and_center_columns_job_test.rb b/test/jobs/update_box_area_and_center_columns_job_test.rb new file mode 100644 index 0000000000..e0824a41c7 --- /dev/null +++ b/test/jobs/update_box_area_and_center_columns_job_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class UpdateBoxAreaAndCenterColumnsJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end From 0559573b0da88007e57404e99033b4bcca823789 Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Mon, 6 Jan 2025 15:08:53 -0800 Subject: [PATCH 3/8] Use the job, not the command --- config/recurring.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/recurring.yml b/config/recurring.yml index 4204dc8749..75324ae686 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -13,5 +13,5 @@ # schedule: every day at 9am production: repopulate_location_and_observation_center_lat_lng: - command: "Location.update_box_area_and_center_columns" + class: UpdateBoxAreaAndCenterColumnsJob schedule: every saturday at midnight From 5aaa91f092bdcc81a2bba537ffbf92171656307e Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Mon, 6 Jan 2025 15:18:47 -0800 Subject: [PATCH 4/8] Update update_box_area_and_center_columns_job_test.rb --- .../update_box_area_and_center_columns_job_test.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/jobs/update_box_area_and_center_columns_job_test.rb b/test/jobs/update_box_area_and_center_columns_job_test.rb index e0824a41c7..88e4869c95 100644 --- a/test/jobs/update_box_area_and_center_columns_job_test.rb +++ b/test/jobs/update_box_area_and_center_columns_job_test.rb @@ -1,7 +1,11 @@ require "test_helper" class UpdateBoxAreaAndCenterColumnsJobTest < ActiveJob::TestCase - # test "the truth" do - # assert true - # end + def test_update_dry_run + # ? + # assert_difference("Observation.count", 1, + # "Failed to create observation") do + # UpdateBoxAreaAndCenterColumnsJob.perform_now(dry_run: true) + # end + end end From e867ee0063bc64ece5e82a68271fe2abf1c6fa72 Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Mon, 6 Jan 2025 15:19:05 -0800 Subject: [PATCH 5/8] Update update_box_area_and_center_columns_job_test.rb --- test/jobs/update_box_area_and_center_columns_job_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/jobs/update_box_area_and_center_columns_job_test.rb b/test/jobs/update_box_area_and_center_columns_job_test.rb index 88e4869c95..bd79fb3be8 100644 --- a/test/jobs/update_box_area_and_center_columns_job_test.rb +++ b/test/jobs/update_box_area_and_center_columns_job_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "test_helper" class UpdateBoxAreaAndCenterColumnsJobTest < ActiveJob::TestCase From aabceb35e3df7d3f5d9bb2bb7fd9e4eb259e706d Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Thu, 9 Jan 2025 22:59:35 -0800 Subject: [PATCH 6/8] Add tests --- .../update_box_area_and_center_columns_job.rb | 7 +++-- ...te_box_area_and_center_columns_job_test.rb | 29 +++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/app/jobs/update_box_area_and_center_columns_job.rb b/app/jobs/update_box_area_and_center_columns_job.rb index f629a9cf42..f3a24de84c 100644 --- a/app/jobs/update_box_area_and_center_columns_job.rb +++ b/app/jobs/update_box_area_and_center_columns_job.rb @@ -3,7 +3,8 @@ class UpdateBoxAreaAndCenterColumnsJob < ApplicationJob queue_as :default - def perform(*args) + def perform(**args) + args ||= {} log("Starting UpdateBoxAreaAndCenterColumnsJob.perform") # This should be zero, but count just in case. loc_count = Location.where(box_area: nil).count @@ -13,7 +14,7 @@ def perform(*args) obs_count = Observation.in_box_of_max_area.where(location_lat: nil).count log("Found #{obs_count} observations where the associated location was " \ "small enough, but the obs didn't have a location_lat/lng.") - return if args[:dry_run] + return [loc_count, obs_count] if args[:dry_run] # Do the update. This returns counts of locations/observations updated. loc_updated, obs_centered, obs_center_nulled = @@ -22,5 +23,7 @@ def perform(*args) log("Updated #{loc_updated} locations' box_area and center_lat/lng.") log("Updated #{obs_centered} observations' location_lat/lng.") log("Nulled #{obs_center_nulled} observations' location_lat/lng.") + # Return the values for debugging + [loc_updated, obs_centered, obs_center_nulled] end end diff --git a/test/jobs/update_box_area_and_center_columns_job_test.rb b/test/jobs/update_box_area_and_center_columns_job_test.rb index bd79fb3be8..3717cc190b 100644 --- a/test/jobs/update_box_area_and_center_columns_job_test.rb +++ b/test/jobs/update_box_area_and_center_columns_job_test.rb @@ -3,11 +3,28 @@ require "test_helper" class UpdateBoxAreaAndCenterColumnsJobTest < ActiveJob::TestCase - def test_update_dry_run - # ? - # assert_difference("Observation.count", 1, - # "Failed to create observation") do - # UpdateBoxAreaAndCenterColumnsJob.perform_now(dry_run: true) - # end + def test_update_box_area_and_center_columns + # Check that we have some obs that need updating + assert_not_empty(Observation.in_box_of_max_area.where(location_lat: nil)) + + job = UpdateBoxAreaAndCenterColumnsJob.new + job.perform + + assert_empty(Observation.in_box_of_max_area.where(location_lat: nil)) + # Note: All locations should already have a box_area, so this was nil before + assert_empty(Location.where(box_area: nil)) + end + + def test_update_box_area_dry_run + # Check that we have some obs that need updating + assert_not_empty(Observation.in_box_of_max_area.where(location_lat: nil)) + + job = UpdateBoxAreaAndCenterColumnsJob.new + _loc_count, obs_count = job.perform(dry_run: true) + + # check it again + needs_update = Observation.in_box_of_max_area.where(location_lat: nil) + assert_not_empty(needs_update) + assert_equal(obs_count, needs_update.count) end end From 8bd04a8b224b8a3e851d8dec318568697aebd13b Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Thu, 9 Jan 2025 23:08:58 -0800 Subject: [PATCH 7/8] Update update_box_area_and_center_columns_job_test.rb --- test/jobs/update_box_area_and_center_columns_job_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/jobs/update_box_area_and_center_columns_job_test.rb b/test/jobs/update_box_area_and_center_columns_job_test.rb index 3717cc190b..1781e7aa7d 100644 --- a/test/jobs/update_box_area_and_center_columns_job_test.rb +++ b/test/jobs/update_box_area_and_center_columns_job_test.rb @@ -11,7 +11,7 @@ def test_update_box_area_and_center_columns job.perform assert_empty(Observation.in_box_of_max_area.where(location_lat: nil)) - # Note: All locations should already have a box_area, so this was nil before + # NOTE: All locations may already have a box_area, so probably nil before assert_empty(Location.where(box_area: nil)) end @@ -22,7 +22,7 @@ def test_update_box_area_dry_run job = UpdateBoxAreaAndCenterColumnsJob.new _loc_count, obs_count = job.perform(dry_run: true) - # check it again + # check it again and be sure it did NOT update the obs needs_update = Observation.in_box_of_max_area.where(location_lat: nil) assert_not_empty(needs_update) assert_equal(obs_count, needs_update.count) From 384a1bddec8140f3f568e2e20e093f13e7b0ab98 Mon Sep 17 00:00:00 2001 From: andrew nimmo Date: Thu, 9 Jan 2025 23:38:58 -0800 Subject: [PATCH 8/8] Update recurring.yml Switch to daily --- config/recurring.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/recurring.yml b/config/recurring.yml index 75324ae686..2851f10da5 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -14,4 +14,4 @@ production: repopulate_location_and_observation_center_lat_lng: class: UpdateBoxAreaAndCenterColumnsJob - schedule: every saturday at midnight + schedule: every day at midnight