Skip to content

Commit

Permalink
Refactor report_downloader to run .process
Browse files Browse the repository at this point in the history
  • Loading branch information
maxkadel committed Mar 21, 2024
1 parent b048d92 commit c08cd59
Show file tree
Hide file tree
Showing 6 changed files with 27 additions and 24 deletions.
6 changes: 3 additions & 3 deletions app/models/gobi/isbn_file.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
module Gobi
class IsbnFile
attr_reader :received_items_file
def initialize(received_items_file:)
@received_items_file = received_items_file
def initialize(temp_file:)
@received_items_file = temp_file
end

def process
Expand All @@ -14,7 +14,7 @@ def process
next if isbn_for_report(row:).blank?
write_record(row:)
end
true
Gobi::IsbnReportJob.working_file_name
end

# with items published in the last 5 years
Expand Down
8 changes: 3 additions & 5 deletions app/models/gobi/isbn_report_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ def initialize
@report_downloader = ReportDownloader.new(
sftp: AlmaSftp.new,
file_pattern: 'received_items_published_last_5_years_\d{12}.csv',
input_sftp_base_dir: '/alma/isbns'
input_sftp_base_dir: '/alma/isbns',
process_class: Gobi::IsbnFile
)
end

Expand All @@ -25,11 +26,8 @@ def self.working_file_name
end

def handle(data_set:)
temp_files = @report_downloader.download
create_csv
temp_files.each do |temp_file|
Gobi::IsbnFile.new(received_items_file: temp_file).process
end
@report_downloader.run
report_uploader = ReportUploader.new(
sftp: GobiSftp.new,
working_file_names: [IsbnReportJob.working_file_name],
Expand Down
19 changes: 12 additions & 7 deletions app/models/report_downloader.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,19 @@
# This class is responsible for matching and downloading files from a remote sftp server to local temp files
# It returns an array of temp file paths
class ReportDownloader
attr_reader :sftp, :file_pattern, :input_sftp_base_dir
# sftp should be an instance of an Sftp class - e.g. OclcSftp.new, GobiSftp.new
attr_reader :sftp, :file_pattern, :input_sftp_base_dir, :process_class
# sftp should be an instance of an Sftp class - e.g. OclcSftp.new, GobiFtp.new
# file_pattern is a string that will be used as regex to match files against
# input_sftp_base_dir is the directory to check for files in
def initialize(sftp:, file_pattern:, input_sftp_base_dir:)
def initialize(sftp:, file_pattern:, input_sftp_base_dir:, process_class:)
@sftp = sftp
@file_pattern = file_pattern
@input_sftp_base_dir = input_sftp_base_dir
@process_class = process_class
end

def download
temp_file_paths = []
def run
working_file_names = []
sftp.start do |sftp_conn|
sftp_conn.dir.foreach(input_sftp_base_dir) do |entry|
next unless /#{file_pattern}/.match?(entry.name)
Expand All @@ -23,9 +24,13 @@ def download
# ascii-8bit required for download! to succeed
temp_file = Tempfile.new(encoding: 'ascii-8bit')
sftp_conn.download!(remote_filename, temp_file)
temp_file_paths << temp_file.path
working_file_names << process(temp_file)
end
end
temp_file_paths
working_file_names
end

def process(temp_file)
process_class.new(temp_file:).process
end
end
2 changes: 1 addition & 1 deletion spec/models/gobi/isbn_file_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

RSpec.describe Gobi::IsbnFile, type: :model do
include_context 'gobi_isbn'
let(:isbn_file) { described_class.new(received_items_file: temp_file_one) }
let(:isbn_file) { described_class.new(temp_file: temp_file_one) }

it 'can be processed' do
expect(isbn_file.process).to be
Expand Down
1 change: 0 additions & 1 deletion spec/models/gobi/isbn_report_job_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
RSpec.describe Gobi::IsbnReportJob, type: :model do
include_context 'sftp_gobi_isbn'
let(:isbn_job) { described_class.new }
# let(:new_tsv_path) { File.join('spec', 'fixtures', 'gobi', '2024-03-16-gobi-isbn-updates.tsv') }

it_behaves_like 'a lib job'

Expand Down
15 changes: 8 additions & 7 deletions spec/models/report_downloader_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
described_class.new(
sftp: AlmaSftp.new,
file_pattern: 'received_items_published_last_5_years_\d{12}.csv',
input_sftp_base_dir: '/alma/isbns'
input_sftp_base_dir: '/alma/isbns',
process_class: Gobi::IsbnFile
)
end

Expand All @@ -24,17 +25,17 @@
expect(downloader.input_sftp_base_dir).to eq('/alma/isbns')
end

describe '#download' do
describe '#run' do
include_context 'sftp_gobi_isbn'

it 'downloads matching files' do
temp_files = downloader.download
working_file_names = downloader.run
expect(sftp_session).to have_received(:download!).with(file_full_path_one, temp_file_one)
expect(sftp_session).to have_received(:download!).with(file_full_path_two, temp_file_two)
expect(temp_files).to be_an_instance_of(Array)
expect(temp_files.size).to eq(2)
expect(temp_files.first).to be_an_instance_of(String)
expect(File.exist?(temp_files.first)).to be true
expect(working_file_names).to be_an_instance_of(Array)
expect(working_file_names.size).to eq(2)
expect(working_file_names.first).to be_an_instance_of(String)
expect(File.exist?(File.join('spec', 'fixtures', 'gobi', working_file_names.first))).to be true
end
end
end

0 comments on commit c08cd59

Please sign in to comment.