Merge pull request #122 from hathitrust/update-dependencies

Update Dockerfile, dependencies

Dockerfile

@@ -1,7 +1,7 @@
-FROM ruby:3.1
+FROM ruby:3.3 AS base
+LABEL org.opencontainers.image.source="https://github.com/hathitrust/datasets"
+
 ARG UNAME=app
-ARG UID=1000
-ARG GID=1000
 
 # for rotatelogs
 RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
@@ -11,14 +11,19 @@ RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
 # sdrN for volumes and symlinks
 RUN bash -c 'for i in $(seq 1 24); do ln -s /sdr/$i /sdr$i; done'
 
+WORKDIR /usr/src/app
+ENV BUNDLE_PATH /gems
+ENV RUBYLIB /usr/src/app/lib
+
+FROM base AS production
+
+ARG UID=1000
+ARG GID=1000
 RUN gem install bundler
 RUN groupadd -g $GID -o $UNAME
 RUN useradd -m -d /usr/src/app -u $UID -g $GID -o -s /bin/bash $UNAME
 RUN mkdir -p /gems && chown $UID:$GID /gems
 USER $UNAME
 COPY --chown=$UID:$GID Gemfile* /usr/src/app/
-WORKDIR /usr/src/app
-ENV BUNDLE_PATH /gems
-ENV RUBYLIB /usr/src/app/lib
 RUN bundle install
 COPY --chown=$UID:$GID . /usr/src/app

Gemfile

@@ -1,6 +1,5 @@
 source "https://rubygems.org"
 
-gem "activesupport"
 gem "mysql2"
 gem "puma"
 gem "pairtree", "~> 0.3"
@@ -9,6 +8,8 @@ gem "sequel"
 gem "rubyzip"
 gem "thor"
 gem "rake", "~> 12.3"
+gem "csv"
+gem "base64"
 
 group :development, :test do
   gem "rspec", "~> 3.0"

Gemfile.lock

@@ -1,73 +1,72 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.4.3)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
     ast (2.4.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.7)
     byebug (11.1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
-    connection_pool (2.4.0)
-    diff-lcs (1.5.0)
+    connection_pool (2.4.1)
+    csv (3.3.0)
+    diff-lcs (1.5.1)
     docile (1.4.0)
-    i18n (1.12.0)
-      concurrent-ruby (~> 1.0)
-    json (2.6.3)
+    json (2.7.2)
     language_server-protocol (3.17.0.3)
-    method_source (1.0.0)
-    minitest (5.18.0)
-    mysql2 (0.5.5)
-    nio4r (2.5.8)
+    lint_roller (1.1.0)
+    method_source (1.1.0)
+    mysql2 (0.5.6)
+    nio4r (2.7.1)
     pairtree (0.3.0)
-    parallel (1.22.1)
-    parser (3.2.1.1)
+    parallel (1.24.0)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
+      racc
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    puma (6.2.1)
+    puma (6.4.2)
       nio4r (~> 2.0)
-    rack (2.2.6.4)
+    racc (1.7.3)
+    rack (2.2.9)
     rainbow (3.1.1)
     rake (12.3.3)
     redis (4.8.1)
-    regexp_parser (2.7.0)
-    rexml (3.2.5)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.1)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    regexp_parser (2.9.0)
+    rexml (3.2.6)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.5)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
-    rubocop (1.48.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    rubocop (1.62.1)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.0)
-      parser (>= 3.2.1.0)
-    rubocop-performance (1.16.0)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-ast (1.31.2)
+      parser (>= 3.3.0.4)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     ruby-progressbar (1.13.0)
     rubyzip (2.3.2)
-    sequel (5.66.0)
-    sidekiq (6.5.8)
+    sequel (5.79.0)
+      bigdecimal
+    sidekiq (6.5.12)
       connection_pool (>= 2.2.5, < 3)
       rack (~> 2.0)
       redis (>= 4.5.0, < 5)
@@ -78,22 +77,29 @@ GEM
     simplecov-html (0.12.3)
     simplecov-lcov (0.8.0)
     simplecov_json_formatter (0.1.4)
-    standard (1.25.3)
+    standard (1.35.1)
       language_server-protocol (~> 3.17.0.2)
-      rubocop (~> 1.48.1)
-      rubocop-performance (~> 1.16.0)
-    thor (1.2.1)
-    timecop (0.9.6)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
-    unicode-display_width (2.4.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.62.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.3)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.3.1)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.20.2)
+    thor (1.3.1)
+    timecop (0.9.8)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport
+  base64
   byebug
+  csv
   mysql2
   pairtree (~> 0.3)
   pry
@@ -110,4 +116,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   2.3.25
+   2.5.9

README.md

@@ -53,7 +53,12 @@ The list of changes is filtered into queues.  There is a queue for each subset a
 For each volume in a queue, a job is scheduled to apply the changes to the filesystem.
 
 ## Use
-Scheduled job to be run daily? weekly?
+Deployed via [private ArgoCD control repository](https://github.com/hathitrust/ht_tanka/tree/main/environments/datasets/production)
+
+This creates a set of workers for handling data set jobs, as well as a set of
+cron jobs to generate the dataset full inventory, fetch metadata, queue jobs
+for updating the data set, and compiling and processing the logs generated by
+the workers.
 
 ## Assumptions & Dependencies
 Atomic filesystem moves.  This remains to be tested.