From 4e25eb4cf04d3d5a3125f173ef32b9dbb4ad6ee5 Mon Sep 17 00:00:00 2001 From: gabina Date: Tue, 21 Jan 2025 17:27:40 -0300 Subject: [PATCH 1/8] Pass in right update_service to UpdateCourseWikiTimeslices --- app/services/update_course_stats_timeslice.rb | 4 +- app/services/update_course_wiki_timeslices.rb | 11 ++-- .../update_course_wiki_timeslices_spec.rb | 50 +------------------ 3 files changed, 6 insertions(+), 59 deletions(-) diff --git a/app/services/update_course_stats_timeslice.rb b/app/services/update_course_stats_timeslice.rb index 75b85b1d95..d7960946af 100644 --- a/app/services/update_course_stats_timeslice.rb +++ b/app/services/update_course_stats_timeslice.rb @@ -29,7 +29,7 @@ def initialize(course) import_uploads update_categories update_article_status if should_update_article_status? - @timeslice_errors = UpdateCourseWikiTimeslices.new(@course).run(all_time: @full_update) + UpdateCourseWikiTimeslices.new(@course, update_service: self).run(all_time: @full_update) update_average_pageviews update_caches update_wikidata_stats if wikidata @@ -109,7 +109,7 @@ def log_end_of_update UpdateLogger.update_course(@course, 'start_time' => @start_time.to_datetime, 'end_time' => @end_time.to_datetime, 'sentry_tag_uuid' => sentry_tag_uuid, - 'error_count' => error_count + @timeslice_errors) + 'error_count' => error_count) end def wikidata diff --git a/app/services/update_course_wiki_timeslices.rb b/app/services/update_course_wiki_timeslices.rb index e5795d37e1..75c097e8fc 100644 --- a/app/services/update_course_wiki_timeslices.rb +++ b/app/services/update_course_wiki_timeslices.rb @@ -1,9 +1,6 @@ # frozen_string_literal: true require_dependency "#{Rails.root}/lib/course_revision_updater" -require_dependency "#{Rails.root}/lib/analytics/histogram_plotter" -require_dependency "#{Rails.root}/lib/data_cycle/update_logger" -require_dependency "#{Rails.root}/lib/errors/update_service_error_helper" require_dependency "#{Rails.root}/lib/timeslice_manager" require_dependency "#{Rails.root}/lib/data_cycle/update_debugger" @@ -11,19 +8,17 @@ # It updates all the tracked wikis for the course, from the latest start time for every wiki # up to the end of update (today or end date course). class UpdateCourseWikiTimeslices - include UpdateServiceErrorHelper - - def initialize(course) + def initialize(course, update_service: nil) @course = course @timeslice_manager = TimesliceManager.new(@course) @debugger = UpdateDebugger.new(@course) + @update_service = update_service @wikidata_stats_updater = UpdateWikidataStatsTimeslice.new(@course) if wikidata end def run(all_time:) pre_update(all_time) fetch_data_and_process_timeslices_for_every_wiki(all_time) - error_count end private @@ -89,7 +84,7 @@ def fetch_data(wiki, timeslice_start, timeslice_end) wiki, timeslice_start.strftime('%Y%m%d%H%M%S'), timeslice_end.strftime('%Y%m%d%H%M%S'), - update_service: self) + update_service: @update_service) # Only for wikidata project, fetch wikidata stats return unless wiki.project == 'wikidata' && @revisions.present? diff --git a/spec/services/update_course_wiki_timeslices_spec.rb b/spec/services/update_course_wiki_timeslices_spec.rb index 4c3b25af53..69bf6281a7 100644 --- a/spec/services/update_course_wiki_timeslices_spec.rb +++ b/spec/services/update_course_wiki_timeslices_spec.rb @@ -159,7 +159,7 @@ expected_dates.each do |start_time, end_time| expected_wikis.each do |wiki| expect(CourseRevisionUpdater).to receive(:fetch_revisions_and_scores_for_wiki) - .with(course, wiki, start_time, end_time, update_service: updater) + .with(course, wiki, start_time, end_time, update_service: nil) .once end end @@ -169,52 +169,4 @@ end end end - - context 'sentry course update error tracking' do - let(:flags) { { debug_updates: true } } - let(:user) { create(:user, username: 'Ragesoss') } - - before do - travel_to Date.new(2018, 11, 28) - course.campaigns << Campaign.first - JoinCourse.new(course:, user:, role: 0) - end - - it 'tracks update errors properly in Replica' do - allow(Sentry).to receive(:capture_exception) - - # Raising errors only in Replica - stub_request(:any, %r{https://replica-revision-tools.wmcloud.org/.*}).to_raise(Errno::ECONNREFUSED) - VCR.use_cassette 'course_update/replica' do - subject - end - sentry_tag_uuid = updater.sentry_tag_uuid - - # Checking whether Sentry receives correct error and tags as arguments - expect(Sentry).to have_received(:capture_exception).exactly(5).times.with( - Errno::ECONNREFUSED, anything - ) - expect(Sentry).to have_received(:capture_exception) - .exactly(5).times.with anything, hash_including(tags: { update_service_id: sentry_tag_uuid, - course: course.slug }) - end - - it 'tracks update errors properly in LiftWing' do - allow(Sentry).to receive(:capture_exception) - - # Raising errors only in LiftWing - stub_request(:any, %r{https://api.wikimedia.org/service/lw.*}).to_raise(Faraday::ConnectionFailed) - VCR.use_cassette 'course_update/lift_wing_api' do - subject - end - sentry_tag_uuid = updater.sentry_tag_uuid - - # Checking whether Sentry receives correct error and tags as arguments - expect(Sentry).to have_received(:capture_exception) - .exactly(2).times.with(Faraday::ConnectionFailed, anything) - expect(Sentry).to have_received(:capture_exception) - .exactly(2).times.with anything, hash_including(tags: { update_service_id: sentry_tag_uuid, - course: course.slug }) - end - end end From fa542bb5594ab21e7f9565638111e7078d1c0831 Mon Sep 17 00:00:00 2001 From: gabina Date: Tue, 21 Jan 2025 18:20:40 -0300 Subject: [PATCH 2/8] Pass in update_service to UpdateCourseWikiTimeslices --- app/services/prepare_timeslices.rb | 5 +++-- app/services/update_course_wiki_timeslices.rb | 2 +- app/services/update_timeslices_course_user.rb | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/services/prepare_timeslices.rb b/app/services/prepare_timeslices.rb index bdde16e712..10ebcb1ff1 100644 --- a/app/services/prepare_timeslices.rb +++ b/app/services/prepare_timeslices.rb @@ -4,10 +4,11 @@ # Ensures that the necessary timeslices are created prior to a new update # of the course statistics. class PrepareTimeslices - def initialize(course) + def initialize(course, update_service: nil) @course = course @timeslice_manager = TimesliceManager.new(@course) @debugger = UpdateDebugger.new(@course) + @update_service = update_service end # Deletes all existing timeslices and recreates them from scratch. @@ -32,7 +33,7 @@ def adjust_timeslices end # Execute update tasks in a specific order UpdateTimeslicesCourseWiki.new(@course).run - UpdateTimeslicesCourseUser.new(@course).run + UpdateTimeslicesCourseUser.new(@course, update_service: @update_service).run UpdateTimeslicesUntrackedArticle.new(@course).run UpdateTimeslicesCourseDate.new(@course).run UpdateTimeslicesScopedArticle.new(@course).run diff --git a/app/services/update_course_wiki_timeslices.rb b/app/services/update_course_wiki_timeslices.rb index 75c097e8fc..4bc06ac954 100644 --- a/app/services/update_course_wiki_timeslices.rb +++ b/app/services/update_course_wiki_timeslices.rb @@ -25,7 +25,7 @@ def run(all_time:) def pre_update(from_scratch) @debugger.log_update_progress :pre_update_start - prepare_timeslices = PrepareTimeslices.new(@course) + prepare_timeslices = PrepareTimeslices.new(@course, update_service: @update_service) from_scratch ? prepare_timeslices.recreate_timeslices : prepare_timeslices.adjust_timeslices @debugger.log_update_progress :pre_update_finish end diff --git a/app/services/update_timeslices_course_user.rb b/app/services/update_timeslices_course_user.rb index a05a40c83a..74a6084b8f 100644 --- a/app/services/update_timeslices_course_user.rb +++ b/app/services/update_timeslices_course_user.rb @@ -5,9 +5,10 @@ require_dependency "#{Rails.root}/lib/revision_data_manager" class UpdateTimeslicesCourseUser - def initialize(course) + def initialize(course, update_service: nil) @course = course @timeslice_manager = TimesliceManager.new(course) + @update_sercice = update_service end def run @@ -44,7 +45,7 @@ def add_user_ids(user_ids) def fetch_users_revisions_for_wiki(wiki, user_ids) users = User.find(user_ids) - manager = RevisionDataManager.new(wiki, @course) + manager = RevisionDataManager.new(wiki, @course, update_service: @update_service) # Fetch the revisions for users for the complete period revisions = manager.fetch_revision_data_for_users(users, @course.start.strftime('%Y%m%d%H%M%S'), From 94981f970be51aca3c9e5d8ae56118fad99bdc62 Mon Sep 17 00:00:00 2001 From: gabina Date: Tue, 21 Jan 2025 18:48:30 -0300 Subject: [PATCH 3/8] Pass in the same debugger instance --- app/services/prepare_timeslices.rb | 4 ++-- app/services/update_course_stats_timeslice.rb | 3 ++- app/services/update_course_wiki_timeslices.rb | 6 +++--- spec/services/update_course_wiki_timeslices_spec.rb | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/services/prepare_timeslices.rb b/app/services/prepare_timeslices.rb index 10ebcb1ff1..8ecd5272f2 100644 --- a/app/services/prepare_timeslices.rb +++ b/app/services/prepare_timeslices.rb @@ -4,10 +4,10 @@ # Ensures that the necessary timeslices are created prior to a new update # of the course statistics. class PrepareTimeslices - def initialize(course, update_service: nil) + def initialize(course, debugger, update_service: nil) @course = course @timeslice_manager = TimesliceManager.new(@course) - @debugger = UpdateDebugger.new(@course) + @debugger = debugger @update_service = update_service end diff --git a/app/services/update_course_stats_timeslice.rb b/app/services/update_course_stats_timeslice.rb index d7960946af..55cc87ff80 100644 --- a/app/services/update_course_stats_timeslice.rb +++ b/app/services/update_course_stats_timeslice.rb @@ -29,7 +29,8 @@ def initialize(course) import_uploads update_categories update_article_status if should_update_article_status? - UpdateCourseWikiTimeslices.new(@course, update_service: self).run(all_time: @full_update) + UpdateCourseWikiTimeslices.new(@course, @debugger, + update_service: self).run(all_time: @full_update) update_average_pageviews update_caches update_wikidata_stats if wikidata diff --git a/app/services/update_course_wiki_timeslices.rb b/app/services/update_course_wiki_timeslices.rb index 4bc06ac954..eab8a0e0e6 100644 --- a/app/services/update_course_wiki_timeslices.rb +++ b/app/services/update_course_wiki_timeslices.rb @@ -8,10 +8,10 @@ # It updates all the tracked wikis for the course, from the latest start time for every wiki # up to the end of update (today or end date course). class UpdateCourseWikiTimeslices - def initialize(course, update_service: nil) + def initialize(course, debugger, update_service: nil) @course = course @timeslice_manager = TimesliceManager.new(@course) - @debugger = UpdateDebugger.new(@course) + @debugger = debugger @update_service = update_service @wikidata_stats_updater = UpdateWikidataStatsTimeslice.new(@course) if wikidata end @@ -25,7 +25,7 @@ def run(all_time:) def pre_update(from_scratch) @debugger.log_update_progress :pre_update_start - prepare_timeslices = PrepareTimeslices.new(@course, update_service: @update_service) + prepare_timeslices = PrepareTimeslices.new(@course, @debugger, update_service: @update_service) from_scratch ? prepare_timeslices.recreate_timeslices : prepare_timeslices.adjust_timeslices @debugger.log_update_progress :pre_update_finish end diff --git a/spec/services/update_course_wiki_timeslices_spec.rb b/spec/services/update_course_wiki_timeslices_spec.rb index 69bf6281a7..a52ed8e5c9 100644 --- a/spec/services/update_course_wiki_timeslices_spec.rb +++ b/spec/services/update_course_wiki_timeslices_spec.rb @@ -9,7 +9,7 @@ end let(:enwiki) { Wiki.get_or_create(language: 'en', project: 'wikipedia') } let(:wikidata) { Wiki.get_or_create(language: nil, project: 'wikidata') } - let(:updater) { described_class.new(course) } + let(:updater) { described_class.new(course, UpdateDebugger.new(course)) } let(:subject) { updater.run(all_time: false) } let(:flags) { nil } let(:user) { create(:user, username: 'Ragesoss') } From 264264eea5c2518aefa447da3b4effd71eadf7e5 Mon Sep 17 00:00:00 2001 From: gabina Date: Wed, 22 Jan 2025 10:52:19 -0300 Subject: [PATCH 4/8] Add some basic logs through Rails.logger to identify when course metadata changed --- app/services/update_timeslices_course_date.rb | 8 ++++++++ app/services/update_timeslices_course_user.rb | 4 ++++ app/services/update_timeslices_course_wiki.rb | 4 ++++ app/services/update_timeslices_scoped_article.rb | 2 ++ 4 files changed, 18 insertions(+) diff --git a/app/services/update_timeslices_course_date.rb b/app/services/update_timeslices_course_date.rb index 3384407e58..cf30ecda8c 100644 --- a/app/services/update_timeslices_course_date.rb +++ b/app/services/update_timeslices_course_date.rb @@ -49,12 +49,16 @@ def update_timeslices_if_end_date_changed def add_wiki_timeslices_up_to_new_end_date(wiki) mark_old_last_wiki_timeslce_as_needs_update(wiki) + Rails.logger.info "UpdateTimeslicesCourseDate: Adding data up to: #{@course.end}" + @timeslice_manager.create_wiki_timeslices_up_to_new_course_end_date(wiki) end def remove_timeslices_prior_to_start_date mark_new_first_timeslce_as_needs_update + Rails.logger.info "UpdateTimeslicesCourseDate: Removing data prior to: #{@course.start}" + # Delete course and course user timeslices @timeslice_manager.delete_course_wiki_timeslices_prior_to_start_date @timeslice_manager.delete_course_user_wiki_timeslices_prior_to_start_date @@ -66,6 +70,8 @@ def remove_timeslices_prior_to_start_date def remove_timeslices_after_end_date mark_old_last_timeslce_as_needs_update + Rails.logger.info "UpdateTimeslicesCourseDate: Removing data after to: #{@course.end}" + # Delete course and course user timeslices @timeslice_manager.delete_course_wiki_timeslices_after_end_date @timeslice_manager.delete_course_user_wiki_timeslices_after_end_date @@ -77,6 +83,8 @@ def remove_timeslices_after_end_date def add_wiki_timeslices_from_new_start_date(wiki) mark_new_wiki_first_timeslce_as_needs_update(wiki) + Rails.logger.info "UpdateTimeslicesCourseDate: Adding data after to: #{@course.start}" + @timeslice_manager.create_wiki_timeslices_for_new_course_start_date(wiki) end diff --git a/app/services/update_timeslices_course_user.rb b/app/services/update_timeslices_course_user.rb index 74a6084b8f..8183c1e945 100644 --- a/app/services/update_timeslices_course_user.rb +++ b/app/services/update_timeslices_course_user.rb @@ -37,6 +37,8 @@ def run def add_user_ids(user_ids) return if user_ids.empty? + Rails.logger.info "UpdateTimeslicesCourseUser: Adding new users: #{user_ids}" + @course.wikis.each do |wiki| fetch_users_revisions_for_wiki(wiki, user_ids) end @@ -55,6 +57,8 @@ def fetch_users_revisions_for_wiki(wiki, user_ids) def remove_courses_users(user_ids) return if user_ids.empty? + + Rails.logger.info "UpdateTimeslicesCourseUser: Removing old users: #{user_ids}" # Delete course user wiki timeslices for the deleted users @timeslice_manager.delete_course_user_timeslices_for_deleted_course_users user_ids diff --git a/app/services/update_timeslices_course_wiki.rb b/app/services/update_timeslices_course_wiki.rb index 122da9738f..3549c2a82a 100644 --- a/app/services/update_timeslices_course_wiki.rb +++ b/app/services/update_timeslices_course_wiki.rb @@ -30,6 +30,8 @@ def run private def remove_courses_wikis(wiki_ids) + return if wiki_ids.empty? + Rails.logger.info { "UpdateTimeslicesCourseWiki: Deleting wikis: #{wiki_ids}" } # Delete timeslices for the deleted wikis @timeslice_manager.delete_timeslices_for_deleted_course_wikis wiki_ids # Delete articles courses @@ -37,7 +39,9 @@ def remove_courses_wikis(wiki_ids) end def add_courses_wikis(wiki_ids) + return if wiki_ids.empty? wikis = Wiki.where(id: wiki_ids) + Rails.logger.info { "UpdateTimeslicesCourseWiki: Adding wikis: #{wiki_ids}" } # Create course wiki timeslice records for new wikis @timeslice_manager.create_timeslices_for_new_course_wiki_records(wikis) end diff --git a/app/services/update_timeslices_scoped_article.rb b/app/services/update_timeslices_scoped_article.rb index e28933eb6f..9e7e8ccba4 100644 --- a/app/services/update_timeslices_scoped_article.rb +++ b/app/services/update_timeslices_scoped_article.rb @@ -42,6 +42,8 @@ def run def reset(article_ids) return if article_ids.empty? + Rails.logger.info "UpdateTimeslicesUntrackedArticle: Resetting #{@article_ids}" + # Mark course wiki timeslices to be re-proccesed articles = Article.where(id: article_ids) ArticlesCoursesCleanerTimeslice.reset_specific_articles(@course, articles) From 136cef14e9e21b2fb301a5af680f4db4bbeac9e1 Mon Sep 17 00:00:00 2001 From: gabina Date: Wed, 22 Jan 2025 13:21:36 -0300 Subject: [PATCH 5/8] Improve basic logs through Rails.logger --- app/services/update_course_wiki_timeslices.rb | 18 +++++++++++++++++- app/services/update_timeslices_course_date.rb | 12 ++++++++---- app/services/update_timeslices_course_user.rb | 6 ++++-- .../update_timeslices_scoped_article.rb | 3 ++- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/app/services/update_course_wiki_timeslices.rb b/app/services/update_course_wiki_timeslices.rb index eab8a0e0e6..0414d00bb7 100644 --- a/app/services/update_course_wiki_timeslices.rb +++ b/app/services/update_course_wiki_timeslices.rb @@ -58,6 +58,9 @@ def fetch_data_and_process_timeslices(wiki, first_start, latest_start) start_date = [current_start, @course.start].max end_date = [current_start + @timeslice_manager.timeslice_duration(wiki) - 1.second, @course.end].min + + log_processing(wiki, start_date, end_date) + fetch_data(wiki, start_date, end_date) process_timeslices(wiki) current_start += @timeslice_manager.timeslice_duration(wiki) @@ -71,6 +74,9 @@ def fetch_data_and_reprocess_timeslices(wiki) to_reprocess.each do |t| start_date = [t.start, @course.start].max end_date = [t.end - 1.second, @course.end].min + + log_reprocessing(wiki, start_date, end_date) + fetch_data(wiki, start_date, end_date) process_timeslices(wiki) end @@ -153,7 +159,17 @@ def wikidata end def log_error(error) - Sentry.capture_message "#{@course.title} update timeslices error: #{error}", + Sentry.capture_message "#{@course.slug} update timeslices error: #{error}", level: 'error' end + + def log_processing(wiki, start_date, end_date) + Rails.logger.info "UpdateCourseWikiTimeslices: Course: #{@course.slug} Wiki: #{wiki.id}.\ + Processing timeslice [#{start_date}, #{end_date}]" + end + + def log_reprocessing(wiki, start_date, end_date) + Rails.logger.info "UpdateCourseWikiTimeslices: Course: #{@course.slug} Wiki: #{wiki.id}.\ + Reprocessing timeslice [#{start_date}, #{end_date}]" + end end diff --git a/app/services/update_timeslices_course_date.rb b/app/services/update_timeslices_course_date.rb index cf30ecda8c..e3b2f8a692 100644 --- a/app/services/update_timeslices_course_date.rb +++ b/app/services/update_timeslices_course_date.rb @@ -49,7 +49,8 @@ def update_timeslices_if_end_date_changed def add_wiki_timeslices_up_to_new_end_date(wiki) mark_old_last_wiki_timeslce_as_needs_update(wiki) - Rails.logger.info "UpdateTimeslicesCourseDate: Adding data up to: #{@course.end}" + Rails.logger.info "UpdateTimeslicesCourseDate: Course: #{@course.slug}\ + Adding data up to: #{@course.end}" @timeslice_manager.create_wiki_timeslices_up_to_new_course_end_date(wiki) end @@ -57,7 +58,8 @@ def add_wiki_timeslices_up_to_new_end_date(wiki) def remove_timeslices_prior_to_start_date mark_new_first_timeslce_as_needs_update - Rails.logger.info "UpdateTimeslicesCourseDate: Removing data prior to: #{@course.start}" + Rails.logger.info "UpdateTimeslicesCourseDate: Course: #{@course.slug}\ + Removing data prior to: #{@course.start}" # Delete course and course user timeslices @timeslice_manager.delete_course_wiki_timeslices_prior_to_start_date @@ -70,7 +72,8 @@ def remove_timeslices_prior_to_start_date def remove_timeslices_after_end_date mark_old_last_timeslce_as_needs_update - Rails.logger.info "UpdateTimeslicesCourseDate: Removing data after to: #{@course.end}" + Rails.logger.info "UpdateTimeslicesCourseDate: Course: #{@course.slug}\ + Removing data after to: #{@course.end}" # Delete course and course user timeslices @timeslice_manager.delete_course_wiki_timeslices_after_end_date @@ -83,7 +86,8 @@ def remove_timeslices_after_end_date def add_wiki_timeslices_from_new_start_date(wiki) mark_new_wiki_first_timeslce_as_needs_update(wiki) - Rails.logger.info "UpdateTimeslicesCourseDate: Adding data after to: #{@course.start}" + Rails.logger.info "UpdateTimeslicesCourseDate: Course: #{@course.slug}\ + Adding data after to: #{@course.start}" @timeslice_manager.create_wiki_timeslices_for_new_course_start_date(wiki) end diff --git a/app/services/update_timeslices_course_user.rb b/app/services/update_timeslices_course_user.rb index 8183c1e945..1813fed0c0 100644 --- a/app/services/update_timeslices_course_user.rb +++ b/app/services/update_timeslices_course_user.rb @@ -37,7 +37,8 @@ def run def add_user_ids(user_ids) return if user_ids.empty? - Rails.logger.info "UpdateTimeslicesCourseUser: Adding new users: #{user_ids}" + Rails.logger.info "UpdateTimeslicesCourseUser: Course: #{@course.slug}\ + Adding new users: #{user_ids}" @course.wikis.each do |wiki| fetch_users_revisions_for_wiki(wiki, user_ids) @@ -58,7 +59,8 @@ def fetch_users_revisions_for_wiki(wiki, user_ids) def remove_courses_users(user_ids) return if user_ids.empty? - Rails.logger.info "UpdateTimeslicesCourseUser: Removing old users: #{user_ids}" + Rails.logger.info "UpdateTimeslicesCourseUser: Course: #{@course.slug}\ + Removing old users: #{user_ids}" # Delete course user wiki timeslices for the deleted users @timeslice_manager.delete_course_user_timeslices_for_deleted_course_users user_ids diff --git a/app/services/update_timeslices_scoped_article.rb b/app/services/update_timeslices_scoped_article.rb index 9e7e8ccba4..980bc605ce 100644 --- a/app/services/update_timeslices_scoped_article.rb +++ b/app/services/update_timeslices_scoped_article.rb @@ -42,7 +42,8 @@ def run def reset(article_ids) return if article_ids.empty? - Rails.logger.info "UpdateTimeslicesUntrackedArticle: Resetting #{@article_ids}" + Rails.logger.info "UpdateTimeslicesScopedArticle: Course: #{@course.slug}\ + Resetting #{@article_ids}" # Mark course wiki timeslices to be re-proccesed articles = Article.where(id: article_ids) From 3592e974aff5138f33a321342c189f52cd7041f8 Mon Sep 17 00:00:00 2001 From: gabina Date: Wed, 22 Jan 2025 15:26:49 -0300 Subject: [PATCH 6/8] Make UpdateCourseWikiTimeslices return number of timeslices processed and reprocessed. --- app/services/update_course_wiki_timeslices.rb | 5 +++++ spec/services/update_course_wiki_timeslices_spec.rb | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/services/update_course_wiki_timeslices.rb b/app/services/update_course_wiki_timeslices.rb index 0414d00bb7..463e189b46 100644 --- a/app/services/update_course_wiki_timeslices.rb +++ b/app/services/update_course_wiki_timeslices.rb @@ -14,11 +14,14 @@ def initialize(course, debugger, update_service: nil) @debugger = debugger @update_service = update_service @wikidata_stats_updater = UpdateWikidataStatsTimeslice.new(@course) if wikidata + @processed_timeslices = 0 + @reprocessed_timeslices = 0 end def run(all_time:) pre_update(all_time) fetch_data_and_process_timeslices_for_every_wiki(all_time) + [@processed_timeslices, @reprocessed_timeslices] end private @@ -64,6 +67,7 @@ def fetch_data_and_process_timeslices(wiki, first_start, latest_start) fetch_data(wiki, start_date, end_date) process_timeslices(wiki) current_start += @timeslice_manager.timeslice_duration(wiki) + @processed_timeslices += 1 end @debugger.log_update_progress "fetch_and_process_timeslices_finish_#{wiki.id}".to_sym end @@ -79,6 +83,7 @@ def fetch_data_and_reprocess_timeslices(wiki) fetch_data(wiki, start_date, end_date) process_timeslices(wiki) + @reprocessed_timeslices += 1 end @debugger.log_update_progress "fetch_and_reprocess_timeslices_finish_#{wiki.id}".to_sym end diff --git a/spec/services/update_course_wiki_timeslices_spec.rb b/spec/services/update_course_wiki_timeslices_spec.rb index a52ed8e5c9..ec469e556b 100644 --- a/spec/services/update_course_wiki_timeslices_spec.rb +++ b/spec/services/update_course_wiki_timeslices_spec.rb @@ -17,7 +17,9 @@ context 'when debugging is not enabled' do it 'posts no Sentry logs' do expect(Sentry).not_to receive(:capture_message) - subject + processed, reprocessed = subject + expect(processed).to eq(7) + expect(reprocessed).to eq(0) end end From f55d8f28a0f168d4a555c0d3466632ff35382bfe Mon Sep 17 00:00:00 2001 From: gabina Date: Wed, 22 Jan 2025 15:38:32 -0300 Subject: [PATCH 7/8] Log the number of processed and reproceseed timeslices --- app/services/update_course_stats_timeslice.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/services/update_course_stats_timeslice.rb b/app/services/update_course_stats_timeslice.rb index 55cc87ff80..a45220b034 100644 --- a/app/services/update_course_stats_timeslice.rb +++ b/app/services/update_course_stats_timeslice.rb @@ -29,8 +29,9 @@ def initialize(course) import_uploads update_categories update_article_status if should_update_article_status? - UpdateCourseWikiTimeslices.new(@course, @debugger, - update_service: self).run(all_time: @full_update) + @processed, @reprocessed = UpdateCourseWikiTimeslices.new(@course, @debugger, + update_service: self) + .run(all_time: @full_update) update_average_pageviews update_caches update_wikidata_stats if wikidata @@ -110,7 +111,9 @@ def log_end_of_update UpdateLogger.update_course(@course, 'start_time' => @start_time.to_datetime, 'end_time' => @end_time.to_datetime, 'sentry_tag_uuid' => sentry_tag_uuid, - 'error_count' => error_count) + 'error_count' => error_count, + 'proccesed' => @processed, + 'reprocessed' => @reprocessed) end def wikidata From 51dd4bbb732e3b1c5c2518eaab809bc4992809d6 Mon Sep 17 00:00:00 2001 From: gabina Date: Wed, 22 Jan 2025 17:20:43 -0300 Subject: [PATCH 8/8] Fix specs --- .../update_course_wiki_timeslices_spec.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/spec/services/update_course_wiki_timeslices_spec.rb b/spec/services/update_course_wiki_timeslices_spec.rb index ec469e556b..d122025525 100644 --- a/spec/services/update_course_wiki_timeslices_spec.rb +++ b/spec/services/update_course_wiki_timeslices_spec.rb @@ -14,11 +14,19 @@ let(:flags) { nil } let(:user) { create(:user, username: 'Ragesoss') } + before do + stub_wiki_validation + travel_to Date.new(2018, 12, 1) + course.campaigns << Campaign.first + course.wikis << Wiki.get_or_create(language: nil, project: 'wikidata') + JoinCourse.new(course:, user:, role: 0) + end + context 'when debugging is not enabled' do it 'posts no Sentry logs' do expect(Sentry).not_to receive(:capture_message) processed, reprocessed = subject - expect(processed).to eq(7) + expect(processed).to eq(14) expect(reprocessed).to eq(0) end end @@ -33,15 +41,6 @@ end context 'when there are revisions' do - before do - stub_wiki_validation - travel_to Date.new(2018, 12, 1) - course.campaigns << Campaign.first - # Create course wiki timeslices manually for wikidata - course.wikis << Wiki.get_or_create(language: nil, project: 'wikidata') - JoinCourse.new(course:, user:, role: 0) - end - it 'updates article course timeslices caches' do VCR.use_cassette 'course_update' do subject