test for IA (mock) download

hathitrust · Oct 15, 2024 · 5e08bc1 · 5e08bc1
1 parent 1ca60e1
commit 5e08bc1
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 2 deletions.
diff --git a/lib/HTFeed/PackageType/IA/Download.pm b/lib/HTFeed/PackageType/IA/Download.pm
@@ -17,12 +17,13 @@ sub links {
 }
 
 package HTFeed::PackageType::IA::Download;
-use Encode qw(decode);
 
-use warnings;
 use strict;
+use warnings;
 
 use base qw(HTFeed::Stage::Download);
+
+use Encode qw(decode);
 use File::Pairtree qw(id2ppath s2ppchars);
 use File::Path qw(make_path);
 use HTFeed::Config qw(get_config);
@@ -45,6 +46,9 @@ sub run {
     $self->{pt_path}           = $pt_path;
     my @noncore_missing        = ();
 
+    my $labels     = {name => 'ia'};
+    my $start_time = $self->{job_metrics}->time;
+
     foreach my $suffix (@$core_package_items) {
         $self->download(suffix => $suffix);
     }
@@ -127,6 +131,13 @@ sub run {
     );
     $self->_set_done();
 
+    my $end_time        = $self->{job_metrics}->time;
+    my $delta_time      = $end_time - $start_time;
+    my $downloaded_size = $self->{job_metrics}->dir_size($pt_path);
+    $self->{job_metrics}->add("ingest_download_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->add("ingest_download_bytes_r_total", $downloaded_size, $labels);
+    $self->{job_metrics}->inc("ingest_download_items_total", $labels);
+
     return $self->succeeded();
 }
 
@@ -144,6 +155,8 @@ sub download {
     # check if it was already downloaded
     return 1 if -e "$self->{pt_path}/$filename";
 
+    die "died from $self->{pt_path}/$filename \n";
+
     foreach my $link (@{$self->get_links()}) {
         next if not defined $link;
         if ($link =~ /$suffix$/ and $link !~ /_bw_$suffix/) {

diff --git a/t/ia.t b/t/ia.t
@@ -9,6 +9,52 @@ use HTFeed::Test::SpecSupport qw(mock_zephir);
 use HTFeed::Test::Support qw(load_db_fixtures);
 use Test::Spec;
 
+context "mock download" => sub {
+    describe "HTFeed::PackageType::IA::Download" => sub {
+	use HTFeed::JobMetrics;
+	it "increments jobmetrics (even as a mock download)" => sub {
+	    my $jm     = HTFeed::JobMetrics->new;
+	    my $objid  = 'ark:/13960/t7kq2zj36';
+	    my $ia_id  = 'ark+=13960=t7kq2zj36';
+	    my $volume = HTFeed::Volume->new(
+		namespace   => 'test',
+		objid       => $objid,
+		packagetype => 'ia'
+	    );
+	    my $download_items = "ingest_download_items_total";
+
+	    $jm->clear;
+	    $volume->{ia_id} = $ia_id;
+	    my $downloader   = HTFeed::PackageType::IA::Download->new(volume => $volume);
+	    my @mock_files   = (
+		'djvu.xml',
+		'files.xml',
+		'jp2.zip',
+		'meta.xml',
+		'scandata.xml',
+		'scanfactors.xml',
+	    );
+	    # Create mock files in the download dir to skip the actual download from IA
+	    foreach my $mock_file (@mock_files) {
+		my $mock_path = join(
+		    "",
+		    $volume->get_download_directory(),
+		    "/",
+		    $ia_id,
+		    "_",
+		    $mock_file
+		);
+		system("touch $mock_path");
+	    }
+
+	    # Check that the $download_items metric increments upon successful download
+	    ok($jm->get_value($download_items) == 0);
+	    $downloader->run();
+	    ok($jm->get_value($download_items) == 1);
+	};
+    };
+};
+
 context "with volume & temporary ingest/preingest/zipfile dirs" => sub {
     my $volume;
     my $objid;