Merge pull request #1528 from ODNZSL/main

Production deploy: Fetch and display sign database from S3 location

.github/workflows/ci.yml

@@ -94,16 +94,19 @@ jobs:
         env:
           DATABASE_URL: postgres://postgres:postgres@localhost:5432/nzsl_test
           DEVISE_SECRET_KEY: anything
+          AWS_REGION: ap-southeast-2
           RAILS_ENV: test
         run: |
+          cp env-example .env
           bundle exec rails db:prepare
 
       - name: Run rspec
         env:
           DATABASE_URL: postgres://postgres:postgres@localhost:5432/nzsl_test
           DEVISE_SECRET_KEY: anything
           NZSL_ONLINE_SECRET_KEY_BASE: anything
-          APP_DOMAIN_NAME: localhost:3000
-          APP_PROTOCOL: http
+          AWS_REGION: ap-southeast-2
           S3_BUCKET_URL: http://s3-ap-southeast-2.amazonaws.com/dummy-fake/
-        run: bundle exec rspec spec
+        run: |
+          cp env-example .env
+          bundle exec rspec spec

Gemfile

@@ -11,6 +11,7 @@ gem 'pg', '~>1.2'
 # Use SQLite to access signs from a Signbank dictionary export
 gem 'sqlite3'
 
+gem 'aws-sdk-s3'
 gem 'bootsnap', '>= 1.1.0', require: false
 gem 'haml'
 gem 'jquery-rails'

Gemfile.lock

@@ -71,6 +71,22 @@ GEM
     ast (2.4.2)
     autoprefixer-rails (10.3.3.0)
       execjs (~> 2)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.878.0)
+    aws-sdk-core (3.190.2)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.8)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.76.0)
+      aws-sdk-core (~> 3, >= 3.188.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.142.0)
+      aws-sdk-core (~> 3, >= 3.189.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.8)
+    aws-sigv4 (1.8.0)
+      aws-eventstream (~> 1, >= 1.0.2)
     babel-source (5.8.35)
     babel-transpiler (0.7.0)
       babel-source (>= 4.0, < 6)
@@ -175,6 +191,7 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
+    jmespath (1.6.2)
     jquery-rails (4.4.0)
       rails-dom-testing (>= 1, < 3)
       railties (>= 4.2.0)
@@ -420,6 +437,7 @@ PLATFORMS
 
 DEPENDENCIES
   autoprefixer-rails
+  aws-sdk-s3
   bootsnap (>= 1.1.0)
   brakeman
   bundle-audit

app/models/signbank/record.rb

@@ -6,6 +6,10 @@ class Record < ApplicationRecord
     # to create test records
     after_initialize { readonly! unless Rails.env.test? }
 
+    def self.database_version
+      connection.execute('PRAGMA user_version').first['user_version']
+    end
+
     ##
     # Useful for testing - related Signbank records don't have primary keys,
     # so in this case we consider them equal if they are the same type and

app/views/shared/_footer.html.erb

@@ -46,17 +46,24 @@
   </div>
   <div class="copyright">
     <div class="small-10 small-centered medium-8 large-6">
-      <% if @footer && @footer.first_part %>
-        <%= link_to((image_tag "https://i.creativecommons.org/l/by-nc-sa/3.0/88x31.png"), "https://creativecommons.org/licenses/by-nc-sa/3.0", target:"_blank" ) %>
-        <div class="footer-link">
-          NZSL Dictionary by
-          <%= link_to "Deaf Studies Research Unit, Victoria University of Wellington", "https://www.victoria.ac.nz/lals/research/projects/dsru.aspx", target: "_blank" %>
-          is licensed under a
-          <%= link_to "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.","https://creativecommons.org/licenses/by-nc-sa/4.0/", target: "_blank" %>
-          <br/>
-          <%= link_to 'View in NZSL', @footer.path %>
-        </div>
+      <%= link_to "https://creativecommons.org/licenses/by-nc-sa/3.0", target:"_blank" do %>
+        <%= image_tag "https://i.creativecommons.org/l/by-nc-sa/3.0/88x31.png", alt: "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License badge" %>
       <% end %>
+
+      <div class="footer-link">
+        NZSL Dictionary by
+        <%= link_to "Deaf Studies Research Unit, Victoria University of Wellington", "https://www.victoria.ac.nz/lals/research/projects/dsru.aspx", target: "_blank" %>
+        is licensed under a
+        <%= link_to "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.","https://creativecommons.org/licenses/by-nc-sa/4.0/", target: "_blank" %>
+
+          <% if (version = Signbank::Record.database_version) %>
+          <br />
+          Database version <%= version %>.
+        <% end %>
+
+        <br/>
+        <%= link_to 'View in NZSL', @footer.path if @footer && @footer.first_part %>
+      </div>
     </div>
   </div>
 </footer>

app/views/signs/_sign_of_the_day.html.haml

@@ -1,4 +1,4 @@
-- if sign.present?
+- if sign.present? && sign.video
   .sign-of-the-day.text-center
     .column-block
       %h3= t('signs.sign_of_the_day')

app/views/signs/search.html.haml

@@ -7,9 +7,10 @@
   - @signs.each do |sign|
     .search-results__card
       - @query[:s] ? query_text = @query[:s].join(" ") : query_text = " "
-      .video-container.video--placeholder
-        %i.fi-play.play-button
-        = videojs_rails sources: { mp4: sign.video }, class: "normal", controlsList:"nodownload", controls: false, preload: "metadata", loop: true
+      - if sign.video
+        .video-container.video--placeholder
+          %i.fi-play.play-button
+          = videojs_rails sources: { mp4: sign.video }, class: "normal", controlsList:"nodownload", controls: false, preload: "metadata", loop: true
       -# %span.search-results--placeholder
       .clickable_link.small-12.small-centered
         %a.div_link{:href => "#{sign_url(sign.id)}"}

app/views/signs/show.html.haml

@@ -28,11 +28,12 @@
             %i.fi-plus
           %span.buttons-action.add-button-text Add to Vocab Sheet
     .videos.small-12.medium-5
-      .video-container
-        %i.fi-play.play-button
-        = videojs_rails sources: { mp4: @sign.video }, class: "main_video video", controlsList:"nodownload", controls: true, preload: "metadata", loop: true
-        = play_video_button('signs.show.in_slow_motion', nil, class: 'float-left slow')
-        = play_video_button('signs.show.at_normal_speed', nil, class: 'float-left normal')
+      - if @sign.video
+        .video-container
+          %i.fi-play.play-button
+          = videojs_rails sources: { mp4: @sign.video }, class: "main_video video", controlsList:"nodownload", controls: true, preload: "metadata", loop: true
+          = play_video_button('signs.show.in_slow_motion', nil, class: 'float-left slow')
+          = play_video_button('signs.show.at_normal_speed', nil, class: 'float-left normal')
       .glosses-container.glosses.small-12.float-left
         %h2.main_gloss= @sign.gloss_main
         %h2.secondary_gloss= @sign.gloss_secondary
@@ -44,6 +45,7 @@
       .examples-container.clearfix.videos.small-12.small-centered.medium-5.medium-uncentered
         %h3= t('signs.show.usage_examples')
         - @sign.examples.each do |example|
+          - next unless example.video
           .typography.videos
             .video-container
               %i.fi-play.play-button

config/initializers/sign_database.rb

@@ -1,8 +1,5 @@
-# Update the dictionary file if it is older than 1 month
-# We update this file in both dictionary modes because our tests
-# expect the database to test across both modes
-path = Rails.root.join('db', 'dictionary.sqlite3')
-Rails.application.load_tasks
-deployed = !Rails.env.development? && !Rails.env.test?
-
-Rake::Task['dictionary:update'].execute if deployed || (!path.exist? || path.mtime <= 1.month.ago)
+Rails.application.reloader.to_prepare do
+  # Update the dictionary file on boot
+  Rails.application.load_tasks
+  Rake::Task['dictionary:update'].execute
+end

env-example

@@ -1,4 +1,7 @@
 S3_BUCKET_URL: "example.s3.url/"
 NZSL_ONLINE_SECRET_KEY_BASE: 62da7bed624d0cbbe3d186166fdd88db5bb3989075a2154cebe3e5ee20a4f2a2d540865309958346b7b43799d461be2b37c6e27d1fd6ca03b1f59622c5ccc402
 APP_DOMAIN_NAME: "localhost:3000"
-APP_PROTOCOL: "http"
+APP_PROTOCOL: "http"
+
+# The latest public release
+DICTIONARY_DATABASE_S3_LOCATION="s3://nzsl-signbank-media-production/dictionary-exports/nzsl.db"

lib/tasks/dictionary.rake

@@ -1,29 +1,47 @@
-namespace :dictionary do
+namespace :dictionary do # rubocop:disable Metrics/BlockLength
   desc 'Updates the NZSL dictionary packaged with the application to the latest release from Signbank'
   task :update do # rubocop:disable Rails/RakeEnvironment - we need to place this file before the app can start
-    repo = 'odnzsl/nzsl-dictionary-scripts'
-    filename = 'nzsl.db'
-    content_type = 'application/vnd.sqlite3'
-    release_uri = URI::HTTPS.build(host: 'api.github.com', path: "/repos/#{repo}/releases/latest")
-    release = JSON.parse(release_uri.open.read)
-    database_asset = release['assets'].find do |asset|
-      asset['name'] == filename && asset['content_type'] == content_type
-    end
-
-    database_url = database_asset.fetch('browser_download_url')
+    database_s3_location = URI.parse(ENV.fetch('DICTIONARY_DATABASE_S3_LOCATION') || '')
+    raise 'DICTIONARY_DATABASE_S3_LOCATION must be an S3 URI' unless database_s3_location.scheme == 's3'
 
-    File.open('db/new-dictionary.sqlite3', 'wb') do |f|
-      f.write URI.parse(database_url).open.read
-    rescue OpenURI::HTTPError
-      sleep 5 # Wait a few seconds before retrying
-      retry
-    end
+    download_s3_uri(database_s3_location, 'db/new-dictionary.sqlite3')
 
     database = SQLite3::Database.open('db/new-dictionary.sqlite3')
     raise 'Database does not pass integrity check' unless database.integrity_check == [['ok']]
 
+    version = database.get_int_pragma('user_version')
+
     FileUtils.mv('db/new-dictionary.sqlite3', 'db/dictionary.sqlite3')
 
-    puts "Updated db/dictionary.sqlite3 to #{release['name']}"
+    puts "Updated db/dictionary.sqlite3 to #{version}"
+  end
+
+  def s3_client
+    @s3_client ||= Aws::S3::Client.new({
+      region: ENV.fetch('DICTIONARY_AWS_REGION', ENV.fetch('AWS_REGION', nil)),
+      access_key_id: ENV.fetch('DICTIONARY_AWS_ACCESS_KEY_ID', nil),
+      secret_access_key: ENV.fetch('DICTIONARY_AWS_SECRET_ACCESS_KEY', nil)
+    }.compact)
+  end
+
+  def download_s3_uri(s3_uri, target)
+    bucket = s3_uri.host
+    key = s3_uri.path[1..]
+
+    begin
+      s3_client.get_object({ bucket:, key: }, target:)
+    rescue Aws::Errors::MissingCredentialsError,
+           Aws::Sigv4::Errors::MissingCredentialsError,
+           Aws::S3::Errors::ServiceError
+
+      # Fallback to public-URL download over HTTP if credentials are not provided or invalid.
+      # TODO use aws-sdk to leverage aws-client optimizations once unsigned requests are supported:
+      # https://github.com/aws/aws-sdk-ruby/issues/1149
+      public_url = URI.parse(Aws::S3::Bucket.new(bucket, credentials: 0).object(key).public_url)
+      Net::HTTP.start(public_url.host, public_url.port, use_ssl: true) do |http|
+        response = http.get(public_url.request_uri).tap(&:value)
+        File.binwrite(target, response.body)
+      end
+    end
   end
 end

spec/models/signbank/record_spec.rb

@@ -0,0 +1,14 @@
+require 'rails_helper'
+
+RSpec.describe Signbank::Record do
+  describe '.database_version' do
+    it 'returns the user version of the database' do
+      version = 42
+      allow(Signbank::Record.connection).to receive(:execute)
+        .with('PRAGMA user_version')
+        .and_return([{ 'user_version' => version }])
+
+      expect(Signbank::Record.database_version).to eq(version)
+    end
+  end
+end

spec/views/shared/_footer.html.erb_spec.rb

@@ -0,0 +1,15 @@
+require 'rails_helper'
+
+RSpec.describe 'shared/_footer.html.erb' do
+  it 'renders the database version when available' do
+    allow(Signbank::Record).to receive(:database_version).and_return(123_456)
+    render partial: 'shared/footer', locals: { last: false }
+    expect(rendered).to include 'Database version 123456'
+  end
+
+  it 'does not render the database version if it is unavailable' do
+    allow(Signbank::Record).to receive(:database_version).and_return(nil)
+    render partial: 'shared/footer', locals: { last: false }
+    expect(rendered).not_to include 'Database version 123456'
+  end
+end