From b9279702629404c6c74e044ddd2eaa7fde8575d5 Mon Sep 17 00:00:00 2001 From: aidewoode Date: Thu, 11 Jul 2024 09:44:54 +0800 Subject: [PATCH] Fix parallel sync can not clear records when files was deleted --- app/jobs/media_sync_all_job.rb | 13 ++----------- app/jobs/media_sync_job.rb | 17 ++++++++++------- test/jobs/media_sync_all_job_test.rb | 4 +++- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/app/jobs/media_sync_all_job.rb b/app/jobs/media_sync_all_job.rb index 4fd19c72..08e0aaf1 100644 --- a/app/jobs/media_sync_all_job.rb +++ b/app/jobs/media_sync_all_job.rb @@ -2,23 +2,14 @@ class MediaSyncAllJob < MediaSyncJob def perform(dir = Setting.media_path) - parallel_processor_count = self.class.parallel_processor_count - file_paths = MediaFile.file_paths(dir) - file_md5_hashes = Parallel.map(file_paths, in_processes: parallel_processor_count) do |file_path| + file_md5_hashes = Parallel.map(file_paths, in_processes: self.class.parallel_processor_count) do |file_path| MediaFile.get_md5_hash(file_path, with_mtime: true) end existing_songs = Song.where(md5_hash: file_md5_hashes) added_file_paths = file_paths - existing_songs.pluck(:file_path) - - return if added_file_paths.blank? - - grouped_file_paths = (parallel_processor_count > 0) ? added_file_paths.in_groups(parallel_processor_count, false).compact_blank : [added_file_paths] - - added_song_hashes = Parallel.map(grouped_file_paths, in_processes: parallel_processor_count) do |paths| - Media.sync(:added, paths) - end.flatten.compact + added_song_hashes = added_file_paths.blank? ? [] : parallel_sync(:added, added_file_paths).flatten.compact Media.clean_up(added_song_hashes + existing_songs.pluck(:md5_hash)) end diff --git a/app/jobs/media_sync_job.rb b/app/jobs/media_sync_job.rb index ab4b21ba..5c3a9536 100644 --- a/app/jobs/media_sync_job.rb +++ b/app/jobs/media_sync_job.rb @@ -18,13 +18,7 @@ class MediaSyncJob < ApplicationJob end def perform(type, file_paths = []) - parallel_processor_count = self.class.parallel_processor_count - - grouped_file_paths = (parallel_processor_count > 0) ? file_paths.in_groups(parallel_processor_count, false).compact_blank : [file_paths] - - Parallel.each grouped_file_paths, in_processes: parallel_processor_count do |paths| - Media.sync(type, paths) - end + parallel_sync(type, file_paths) end def self.parallel_processor_count @@ -34,6 +28,15 @@ def self.parallel_processor_count private + def parallel_sync(type, file_paths) + parallel_processor_count = self.class.parallel_processor_count + grouped_file_paths = (parallel_processor_count > 0) ? file_paths.in_groups(parallel_processor_count, false).compact_blank : [file_paths] + + Parallel.each grouped_file_paths, in_processes: parallel_processor_count do |paths| + Media.sync(type, paths) + end + end + def before_sync Media.syncing = true end diff --git a/test/jobs/media_sync_all_job_test.rb b/test/jobs/media_sync_all_job_test.rb index 5d1371d7..006eeabb 100644 --- a/test/jobs/media_sync_all_job_test.rb +++ b/test/jobs/media_sync_all_job_test.rb @@ -74,7 +74,10 @@ class MediaSyncAllJobTest < ActiveJob::TestCase end test "should clear records on database when delete file" do + clear_media_data + create_tmp_dir(from: Setting.media_path) do |tmp_dir| + MediaSyncAllJob.perform_now(tmp_dir) File.delete File.join(tmp_dir, "artist2_album3.ogg") File.delete File.join(tmp_dir, "artist2_album3.wav") File.delete File.join(tmp_dir, "artist2_album3.opus") @@ -82,7 +85,6 @@ class MediaSyncAllJobTest < ActiveJob::TestCase File.delete File.join(tmp_dir, "artist2_album3.wma") MediaSyncAllJob.perform_now(tmp_dir) - assert_nil Song.find_by(name: "ogg_sample") assert_nil Song.find_by(name: "wav_sample") assert_nil Song.find_by(name: "opus_sample")