From cd804906601ae578c596b308bd970d6f68cb98ad Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Fri, 6 Oct 2023 16:27:40 -0400 Subject: [PATCH 01/22] Update Dockerfile, environments, schema --- Dockerfile | 11 +++++++++-- config/environments/development.rb | 3 ++- db/schema.rb | 23 +++++++++++------------ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index c4bbac4..ee030fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION=2.7.3 +ARG RUBY_VERSION=3.1.2 FROM ruby:$RUBY_VERSION-alpine ## Install dependencies: @@ -28,8 +28,15 @@ ENV PATH="/app/ld4p/qa_server-webapp:$PATH" ENV RAILS_ROOT="/app/ld4p/qa_server-webapp" COPY Gemfile Gemfile.lock ./ + RUN gem update --system -RUN bundle install + +## TEMPORARY TO UPDATE RAILS +## Remove this after updating qa_server with new rails version, +## and uncomment RUN bundle install +COPY qa_server /tmp/qa_server +RUN bundle config --local path /tmp/qa_server && bundle install +#RUN bundle install COPY . . RUN bundle exec rake assets:precompile diff --git a/config/environments/development.rb b/config/environments/development.rb index 004b207..09c6a74 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -54,7 +54,8 @@ config.assets.quiet = true # Raises error for missing translations - config.action_view.raise_on_missing_translations = true + # raise_on_missing_translations DEPRECATED IN RAILS 7 + # config.action_view.raise_on_missing_translations = true # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. diff --git a/db/schema.rb b/db/schema.rb index 5b10243..b1f4599 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2,18 +2,17 @@ # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_02_18_125146) do - - create_table "performance_history", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| - t.datetime "dt_stamp" +ActiveRecord::Schema[7.0].define(version: 2020_02_18_125146) do + create_table "performance_history", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.datetime "dt_stamp", precision: nil t.string "authority" t.integer "action" t.integer "size_bytes" @@ -25,7 +24,7 @@ t.index ["action"], name: "index_performance_history_on_action" end - create_table "scenario_run_history", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "scenario_run_history", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| t.bigint "scenario_run_registry_id" t.integer "status", default: 2 t.string "authority_name" @@ -45,8 +44,8 @@ t.index ["url"], name: "index_scenario_run_history_on_url" end - create_table "scenario_run_registry", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| - t.datetime "dt_stamp" + create_table "scenario_run_registry", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.datetime "dt_stamp", precision: nil end end From 1520646c69198c0fd08e41016043f5c85f72c1ad Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Fri, 6 Oct 2023 16:44:58 -0400 Subject: [PATCH 02/22] Pin mariadb version to 10.6 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d838bb3..2e29f96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: - mysql mysql: - image: mariadb + image: mariadb:10.6 # Not specifying version causes a runtime error: "undefined method `encoding' for -0.0:BigDecimal" restart: always container_name: ld4p-qa_server-mysql environment: From 41dc735009726e02121a487b13047c375a8058b8 Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Fri, 6 Oct 2023 17:22:10 -0400 Subject: [PATCH 03/22] Update Gemfile & lock --- Gemfile | 35 ++- Gemfile.lock | 713 +++++++++++++++++++++++++++------------------------ 2 files changed, 406 insertions(+), 342 deletions(-) diff --git a/Gemfile b/Gemfile index 0ee6cd2..cb1f1ec 100644 --- a/Gemfile +++ b/Gemfile @@ -1,16 +1,16 @@ source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '2.7.3' +ruby '3.1.2' # rubocop:disable Bundler/OrderedGems ## Gems adds by `rails new` # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 5.2.5' +gem 'rails', '~> 7.0.8' # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use Puma as the app server -gem 'puma', '~> 5.6.4' +gem 'puma', '>= 5.6.4', '~>6.4.0' # Use SCSS for stylesheets gem 'sass-rails', '~> 5.0' # Use Uglifier as compressor for JavaScript assets @@ -19,11 +19,11 @@ gem 'uglifier', '>= 1.3.0' # gem 'mini_racer', platforms: :ruby # Use CoffeeScript for .coffee assets and views -gem 'coffee-rails', '~> 4.2' +gem 'coffee-rails', '~> 5.0.0' # Turbolinks makes navigating your web application faster. Read more: https://github.com/turbolinks/turbolinks gem 'turbolinks', '~> 5' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder -gem 'jbuilder', '~> 2.5' +gem 'jbuilder', '~> 2.10' # Use Redis adapter to run Action Cable in production # gem 'redis', '~> 4.0' # Use ActiveModel has_secure_password @@ -36,7 +36,7 @@ gem 'jbuilder', '~> 2.5' # gem 'capistrano-rails', group: :development # Reduces boot times through caching; required in config/boot.rb -gem 'bootsnap', '>= 1.1.0', require: false +gem 'bootsnap', '~> 1.16', require: false group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console @@ -46,7 +46,7 @@ end group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' - gem 'listen', '>= 3.0.5', '< 3.2' + gem 'listen', '>= 3.0.5', '< 3.9' # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring gem 'spring' gem 'spring-watcher-listen', '~> 2.0.0' @@ -65,9 +65,19 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] ## Gems manually added to for qa and qa_server engines # Required gems for QA and linked data access -gem 'qa_server', '~> 7.5' -gem 'qa', '~> 5.5' +# gem 'qa_server', '~> 7.5' + +# TEMPORARY TO UPDATE RAILS +# Remove these two qa_server entries after updating qa_server with new rails version. +# First for building in Docker: +gem 'qa_server', path: '/tmp/qa_server' +# Next line for building as a standard Rails project sans Docker. +# This assumes that the remote qa_server is a subdirectory of the main project (which makes +# things so much easier when we are using Docker). +# gem 'qa_server', path: 'qa_server' +gem 'qa', '~> 5.10' gem 'linkeddata' +gem 'psych', '~> 5.1' ## Gems added for application customization # support for .env file @@ -87,7 +97,12 @@ group :development, :integration, :test do end group :development, :integration do - gem 'xray-rails' # overlay showing which files are contributing to the UI + # gem 'xray-rails' # overlay showing which files are contributing to the UI + # This gem doesn't work with Ruby 3.x unless a special branch is used (see + # https://github.com/brentd/xray-rails/pull/108). There is a PR to merge the fix + # into the master branch, but the gem's authors don't seem to be in a hurry to + # finish the work. + gem "xray-rails", git: "https://github.com/brentd/xray-rails.git", branch: "bugs/ruby-3.0.0" end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 51ec6f9..fcb26ce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,62 +1,101 @@ +GIT + remote: https://github.com/brentd/xray-rails.git + revision: f121814718c9907b20058dc9357b80a53afab821 + branch: bugs/ruby-3.0.0 + specs: + xray-rails (0.3.2) + rails (>= 3.1.0) + +PATH + remote: qa_server + specs: + qa_server (7.9.2) + gruff + rails (~> 7.0) + sass-rails (~> 5.0) + useragent + GEM remote: https://rubygems.org/ specs: - actioncable (5.2.6) - actionpack (= 5.2.6) + actioncable (7.0.8) + actionpack (= 7.0.8) + activesupport (= 7.0.8) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.6) - actionpack (= 5.2.6) - actionview (= 5.2.6) - activejob (= 5.2.6) + actionmailbox (7.0.8) + actionpack (= 7.0.8) + activejob (= 7.0.8) + activerecord (= 7.0.8) + activestorage (= 7.0.8) + activesupport (= 7.0.8) + mail (>= 2.7.1) + net-imap + net-pop + net-smtp + actionmailer (7.0.8) + actionpack (= 7.0.8) + actionview (= 7.0.8) + activejob (= 7.0.8) + activesupport (= 7.0.8) mail (~> 2.5, >= 2.5.4) + net-imap + net-pop + net-smtp rails-dom-testing (~> 2.0) - actionpack (5.2.6) - actionview (= 5.2.6) - activesupport (= 5.2.6) - rack (~> 2.0, >= 2.0.8) + actionpack (7.0.8) + actionview (= 7.0.8) + activesupport (= 7.0.8) + rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.6) - activesupport (= 5.2.6) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (7.0.8) + actionpack (= 7.0.8) + activerecord (= 7.0.8) + activestorage (= 7.0.8) + activesupport (= 7.0.8) + globalid (>= 0.6.0) + nokogiri (>= 1.8.5) + actionview (7.0.8) + activesupport (= 7.0.8) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.6) - activesupport (= 5.2.6) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activejob (7.0.8) + activesupport (= 7.0.8) globalid (>= 0.3.6) - activemodel (5.2.6) - activesupport (= 5.2.6) - activerecord (5.2.6) - activemodel (= 5.2.6) - activesupport (= 5.2.6) - arel (>= 9.0) - activerecord-import (1.2.0) - activerecord (>= 3.2) - activestorage (5.2.6) - actionpack (= 5.2.6) - activerecord (= 5.2.6) - marcel (~> 1.0.0) - activesupport (5.2.6) + activemodel (7.0.8) + activesupport (= 7.0.8) + activerecord (7.0.8) + activemodel (= 7.0.8) + activesupport (= 7.0.8) + activerecord-import (1.5.0) + activerecord (>= 4.2) + activestorage (7.0.8) + actionpack (= 7.0.8) + activejob (= 7.0.8) + activerecord (= 7.0.8) + activesupport (= 7.0.8) + marcel (~> 1.0) + mini_mime (>= 1.1.0) + activesupport (7.0.8) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) - amazing_print (1.3.0) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.5) + public_suffix (>= 2.0.2, < 6.0) archive-zip (0.12.0) io-like (~> 0.3.0) - arel (9.0.0) ast (2.4.2) - bcp47 (0.3.3) - i18n - better_errors (2.9.1) - coderay (>= 1.0.0) + base64 (0.1.1) + bcp47_spec (0.2.1) + better_errors (2.10.1) erubi (>= 1.0.0) rack (>= 0.9.0) + rouge (>= 1.0.0) bindex (0.8.1) binding_of_caller (1.0.0) debug_inspector (>= 0.0.1) @@ -66,35 +105,34 @@ GEM rubocop-performance rubocop-rails rubocop-rspec - bootsnap (1.7.7) - msgpack (~> 1.0) + bootsnap (1.16.0) + msgpack (~> 1.2) builder (3.2.4) byebug (11.1.3) - capybara (3.35.3) + capybara (3.39.2) addressable + matrix mini_mime (>= 0.1.3) nokogiri (~> 1.8) rack (>= 1.6.0) rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) - capybara-screenshot (1.0.25) + capybara-screenshot (1.0.26) capybara (>= 1.0, < 4) launchy - childprocess (3.0.0) chromedriver-helper (2.1.1) archive-zip (~> 0.10) nokogiri (~> 1.8) - coderay (1.1.3) - coffee-rails (4.2.2) + coffee-rails (5.0.0) coffee-script (>= 2.2.0) - railties (>= 4.0.0) + railties (>= 5.2.0) coffee-script (2.4.1) coffee-script-source execjs coffee-script-source (1.12.2) - concurrent-ruby (1.1.9) - connection_pool (2.2.5) + concurrent-ruby (1.2.2) + connection_pool (2.4.1) coveralls (0.8.23) json (>= 1.8, < 3) simplecov (~> 0.16.1) @@ -104,280 +142,283 @@ GEM crack (0.4.5) rexml crass (1.0.6) + date (3.3.3) debug_inspector (1.1.0) deprecation (1.1.0) activesupport - diff-lcs (1.4.4) + diff-lcs (1.5.0) docile (1.4.0) - dotenv (2.7.6) - dotenv-rails (2.7.6) - dotenv (= 2.7.6) + dotenv (2.8.1) + dotenv-rails (2.8.1) + dotenv (= 2.8.1) railties (>= 3.2) - ebnf (2.1.3) - amazing_print (~> 1.2) + ebnf (2.4.0) htmlentities (~> 4.3) - rdf (~> 3.1) + rdf (~> 3.3) scanf (~> 1.0) - sxp (~> 1.1) - unicode-types (~> 1.6) - equivalent-xml (0.6.0) - nokogiri (>= 1.4.3) - erubi (1.10.0) - execjs (2.8.1) + sxp (~> 1.3) + unicode-types (~> 1.8) + erubi (1.12.0) + execjs (2.9.1) factory_bot (4.11.1) activesupport (>= 3.0.0) factory_bot_rails (4.11.1) factory_bot (~> 4.11.1) railties (>= 3.0.0) - faker (2.18.0) - i18n (>= 1.6, < 2) - faraday (1.7.0) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0.1) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.1) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - multipart-post (>= 1.2, < 3) + faker (3.2.1) + i18n (>= 1.8.11, < 2) + faraday (2.7.11) + base64 + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - ffi (1.15.3) - geocoder (1.6.7) - globalid (0.5.2) - activesupport (>= 5.0) - gruff (0.14.0) + faraday-net_http (3.0.2) + ffi (1.16.3) + geocoder (1.8.2) + globalid (1.2.1) + activesupport (>= 6.1) + gruff (0.23.0) histogram - rmagick - haml (5.2.2) - temple (>= 0.8.0) + rmagick (>= 5.3) + haml (6.2.3) + temple (>= 0.8.2) + thor tilt hamster (3.0.0) concurrent-ruby (~> 1.0) hashdiff (1.0.1) histogram (0.2.4.1) htmlentities (4.3.4) - i18n (1.8.10) + i18n (1.14.1) concurrent-ruby (~> 1.0) io-like (0.3.1) - jbuilder (2.11.2) + jbuilder (2.11.5) + actionview (>= 5.0.0) activesupport (>= 5.0.0) - json (2.5.1) - json-canonicalization (0.2.1) - json-ld (3.1.9) + json (2.6.3) + json-canonicalization (0.3.2) + json-ld (3.3.0) htmlentities (~> 4.3) - json-canonicalization (~> 0.2) + json-canonicalization (~> 0.3, >= 0.3.2) link_header (~> 0.0, >= 0.0.8) - multi_json (~> 1.14) - rack (~> 2.0) - rdf (~> 3.1) - json-ld-preloaded (3.1.6) - json-ld (~> 3.1) - rdf (~> 3.1) - launchy (2.5.0) - addressable (~> 2.7) - ld-patch (3.1.3) - ebnf (~> 2.1) - rdf (~> 3.1) - rdf-xsd (~> 3.1) - sparql (~> 3.1) - sxp (~> 1.1) - ldpath (1.1.0) + multi_json (~> 1.15) + rack (>= 2.2, < 4) + rdf (~> 3.3) + json-ld-preloaded (3.3.0) + json-ld (~> 3.3) + rdf (~> 3.3) + launchy (2.5.2) + addressable (~> 2.8) + ld-patch (3.3.0) + ebnf (~> 2.4) + rdf (~> 3.3) + rdf-xsd (~> 3.3) + sparql (~> 3.3) + sxp (~> 1.3) + ldpath (1.2.0) nokogiri (~> 1.8) parslet rdf (~> 3.0) + rdf-vocab (~> 3.0) link_header (0.0.8) - linkeddata (3.1.5) - equivalent-xml (~> 0.6) - json-ld (~> 3.1, >= 3.1.9) - json-ld-preloaded (~> 3.1, >= 3.1.5) - ld-patch (~> 3.1, >= 3.1.3) - nokogiri (~> 1.10) - nokogumbo (~> 2.0) - rdf (~> 3.1, >= 3.1.13) - rdf-aggregate-repo (~> 3.1) - rdf-isomorphic (~> 3.1, >= 3.1.1) - rdf-json (~> 3.1) - rdf-microdata (~> 3.1, >= 3.1.2) - rdf-n3 (~> 3.1, >= 3.1.2) - rdf-normalize (~> 0.4) - rdf-ordered-repo (~> 3.1, >= 3.1.1) - rdf-rdfa (~> 3.1, >= 3.1.3) - rdf-rdfxml (~> 3.1, >= 3.1.1) - rdf-reasoner (~> 0.7, >= 0.7.2) - rdf-tabular (~> 3.1, >= 3.1.1) - rdf-trig (~> 3.1, >= 3.1.2) - rdf-trix (~> 3.1) - rdf-turtle (~> 3.1, >= 3.1.3) - rdf-vocab (~> 3.1, >= 3.1.12) - rdf-xsd (~> 3.1) - shacl (~> 0.1, >= 0.1.1) - shex (~> 0.6, >= 0.6.1) - sparql (~> 3.1, >= 3.1.5) - sparql-client (~> 3.1, >= 3.1.2) - listen (3.1.5) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) - ruby_dep (~> 1.2) - lograge (0.11.2) + linkeddata (3.3.1) + json-ld (~> 3.3) + json-ld-preloaded (~> 3.3) + ld-patch (~> 3.3) + nokogiri (~> 1.15, >= 1.15.4) + rdf (~> 3.2, >= 3.2.1) + rdf-aggregate-repo (~> 3.2) + rdf-hamster-repo (~> 3.3) + rdf-isomorphic (~> 3.3) + rdf-json (~> 3.3) + rdf-microdata (~> 3.3) + rdf-n3 (~> 3.3) + rdf-normalize (~> 0.7) + rdf-ordered-repo (~> 3.3) + rdf-rdfa (~> 3.3) + rdf-rdfxml (~> 3.3) + rdf-reasoner (~> 0.9) + rdf-tabular (~> 3.3) + rdf-trig (~> 3.3) + rdf-trix (~> 3.3) + rdf-turtle (~> 3.3) + rdf-vocab (~> 3.3) + rdf-xsd (~> 3.3) + shacl (~> 0.4) + shex (~> 0.8) + sparql (~> 3.3) + sparql-client (~> 3.3) + yaml-ld (~> 0.0) + listen (3.8.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + logger (1.5.3) + lograge (0.13.0) actionpack (>= 4) activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.12.0) + loofah (2.21.3) crass (~> 1.0.2) - nokogiri (>= 1.5.9) - mail (2.7.1) + nokogiri (>= 1.12.0) + mail (2.8.1) mini_mime (>= 0.1.1) - marcel (1.0.1) + net-imap + net-pop + net-smtp + marcel (1.0.2) + matrix (0.4.2) method_source (1.0.0) - mini_mime (1.1.0) - minitest (5.14.4) - msgpack (1.4.2) + mini_mime (1.1.5) + minitest (5.20.0) + msgpack (1.7.2) multi_json (1.15.0) - multipart-post (2.1.1) - mysql2 (0.5.3) - net-http-persistent (4.0.1) + mysql2 (0.5.5) + net-http-persistent (4.0.2) connection_pool (~> 2.2) - nio4r (2.5.8) - nokogiri (1.13.4-x86_64-darwin) - racc (~> 1.4) - nokogiri (1.13.4-x86_64-linux) + net-imap (0.4.0) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.1) + timeout + net-smtp (0.4.0) + net-protocol + nio4r (2.5.9) + nokogiri (1.15.4-x86_64-darwin) racc (~> 1.4) - nokogumbo (2.0.5) - nokogiri (~> 1.8, >= 1.8.4) - parallel (1.20.1) - parser (3.0.2.0) + parallel (1.23.0) + parser (3.2.2.4) ast (~> 2.4.1) + racc parslet (2.0.0) - public_suffix (4.0.6) - puma (5.6.4) + pkg-config (1.5.5) + psych (5.1.0) + stringio + public_suffix (5.0.3) + puma (6.4.0) nio4r (~> 2.0) - qa (5.6.0) + qa (5.10.0) activerecord-import deprecation - faraday + faraday (< 3.0, != 2.0.0) geocoder ldpath nokogiri (~> 1.6) - rails (>= 5.0, < 6.2) + rails (>= 5.0, < 7.1) rdf - qa_server (7.9.2) - gruff - rails (~> 5.2, >= 5.2.5) - sass-rails (~> 5.0) - useragent - racc (1.6.0) - rack (2.2.3) - rack-test (1.1.0) - rack (>= 1.0, < 3) - rails (5.2.6) - actioncable (= 5.2.6) - actionmailer (= 5.2.6) - actionpack (= 5.2.6) - actionview (= 5.2.6) - activejob (= 5.2.6) - activemodel (= 5.2.6) - activerecord (= 5.2.6) - activestorage (= 5.2.6) - activesupport (= 5.2.6) - bundler (>= 1.3.0) - railties (= 5.2.6) - sprockets-rails (>= 2.0.0) + racc (1.7.1) + rack (2.2.8) + rack-test (2.1.0) + rack (>= 1.3) + rails (7.0.8) + actioncable (= 7.0.8) + actionmailbox (= 7.0.8) + actionmailer (= 7.0.8) + actionpack (= 7.0.8) + actiontext (= 7.0.8) + actionview (= 7.0.8) + activejob (= 7.0.8) + activemodel (= 7.0.8) + activerecord (= 7.0.8) + activestorage (= 7.0.8) + activesupport (= 7.0.8) + bundler (>= 1.15.0) + railties (= 7.0.8) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) actionview (>= 5.0.1.rc1) activesupport (>= 5.0.1.rc1) - rails-dom-testing (2.0.3) - activesupport (>= 4.2.0) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.3.0) - loofah (~> 2.3) - railties (5.2.6) - actionpack (= 5.2.6) - activesupport (= 5.2.6) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (7.0.8) + actionpack (= 7.0.8) + activesupport (= 7.0.8) method_source - rake (>= 0.8.7) - thor (>= 0.19.0, < 2.0) - rainbow (3.0.0) + rake (>= 12.2) + thor (~> 1.0) + zeitwerk (~> 2.5) + rainbow (3.1.1) rake (13.0.6) - rb-fsevent (0.11.0) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) - rdf (3.1.15) - hamster (~> 3.0) + rdf (3.3.1) + bcp47_spec (~> 0.2) link_header (~> 0.0, >= 0.0.8) - rdf-aggregate-repo (3.1.0) - rdf (~> 3.1) - rdf-isomorphic (3.1.1) - rdf (~> 3.1) - rdf-json (3.1.0) - rdf (~> 3.1) - rdf-microdata (3.1.3) + rdf-aggregate-repo (3.3.0) + rdf (~> 3.3) + rdf-hamster-repo (3.3.0) + hamster (~> 3.0) + rdf (~> 3.3) + rdf-isomorphic (3.3.0) + rdf (~> 3.3) + rdf-json (3.3.0) + rdf (~> 3.3) + rdf-microdata (3.3.0) htmlentities (~> 4.3) - nokogiri (~> 1.10) - rdf (~> 3.1, >= 3.1.13) - rdf-rdfa (~> 3.1, >= 3.1.3) - rdf-xsd (~> 3.1) - rdf-n3 (3.1.2) - ebnf (~> 2.1) - rdf (~> 3.1, >= 3.1.8) - sparql (~> 3.1, >= 3.1.4) - sxp (~> 1.1) - rdf-normalize (0.4.0) - rdf (~> 3.1) - rdf-ordered-repo (3.1.1) - rdf (~> 3.1) - rdf-rdfa (3.1.3) - haml (~> 5.2) + nokogiri (~> 1.15, >= 1.15.4) + rdf (~> 3.3) + rdf-rdfa (~> 3.3) + rdf-xsd (~> 3.3) + rdf-n3 (3.3.0) + ebnf (~> 2.4) + rdf (~> 3.3) + sparql (~> 3.3) + sxp (~> 1.3) + rdf-normalize (0.7.0) + rdf (~> 3.3) + rdf-ordered-repo (3.3.0) + rdf (~> 3.3) + rdf-rdfa (3.3.0) + haml (~> 6.1) htmlentities (~> 4.3) - rdf (~> 3.1, >= 3.1.13) - rdf-aggregate-repo (~> 3.1) - rdf-vocab (~> 3.1, >= 3.1.11) - rdf-xsd (~> 3.1) - rdf-rdfxml (3.1.1) + rdf (~> 3.3) + rdf-aggregate-repo (~> 3.3) + rdf-vocab (~> 3.3) + rdf-xsd (~> 3.3) + rdf-rdfxml (3.3.0) + builder (~> 3.2, >= 3.2.4) htmlentities (~> 4.3) - rdf (~> 3.1) - rdf-rdfa (~> 3.1) - rdf-xsd (~> 3.1) - rdf-reasoner (0.7.2) - rdf (~> 3.1, >= 3.1.12) - rdf-xsd (~> 3.1) - rdf-tabular (3.1.1) - addressable (~> 2.3) - bcp47 (~> 0.3, >= 0.3.3) - json-ld (~> 3.1) - rdf (~> 3.1) - rdf-vocab (~> 3.1) - rdf-xsd (~> 3.1) - rdf-trig (3.1.2) - ebnf (~> 2.1) - rdf (~> 3.1) - rdf-turtle (~> 3.1) - rdf-trix (3.1.1) - rdf (~> 3.1) - rdf-xsd (~> 3.1) - rdf-turtle (3.1.3) - ebnf (~> 2.1) - rdf (~> 3.1, >= 3.1.8) - rdf-vocab (3.1.14) - rdf (~> 3.1, >= 3.1.12) - rdf-xsd (3.1.1) - rdf (~> 3.1) + rdf (~> 3.3) + rdf-xsd (~> 3.3) + rdf-reasoner (0.9.0) + rdf (~> 3.3) + rdf-xsd (~> 3.3) + rdf-tabular (3.3.0) + addressable (~> 2.8) + bcp47_spec (~> 0.2) + json-ld (~> 3.3) + rdf (~> 3.3) + rdf-vocab (~> 3.3) + rdf-xsd (~> 3.3) + rdf-trig (3.3.0) + ebnf (~> 2.4) + rdf (~> 3.3) + rdf-turtle (~> 3.3) + rdf-trix (3.3.0) + rdf (~> 3.3) + rdf-xsd (~> 3.3) + rdf-turtle (3.3.0) + ebnf (~> 2.4) + rdf (~> 3.3) + rdf-vocab (3.3.0) + rdf (~> 3.3) + rdf-xsd (3.3.0) + rdf (~> 3.3) rexml (~> 3.2) - regexp_parser (2.1.1) - request_store (1.5.0) + regexp_parser (2.8.1) + request_store (1.5.1) rack (>= 1.4) - rexml (3.2.5) - rmagick (4.2.2) + rexml (3.2.6) + rmagick (5.3.0) + pkg-config (~> 1.4) + rouge (4.1.3) rspec-activemodel-mocks (1.1.0) activemodel (>= 3.0) activesupport (>= 3.0) @@ -402,7 +443,7 @@ GEM rspec-mocks (~> 3.9.0) rspec-support (~> 3.9.0) rspec-support (3.9.4) - rspec_junit_formatter (0.4.1) + rspec_junit_formatter (0.6.0) rspec-core (>= 2, < 4, != 2.12.0) rubocop (0.85.1) parallel (~> 1.10) @@ -425,9 +466,8 @@ GEM rubocop (>= 0.82.0) rubocop-rspec (1.41.0) rubocop (>= 0.68.1) - ruby-progressbar (1.11.0) + ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) - ruby_dep (1.5.0) rubyzip (2.3.2) sass (3.7.4) sass-listen (~> 4.0.0) @@ -441,38 +481,41 @@ GEM sprockets-rails (>= 2.0, < 4.0) tilt (>= 1.1, < 3) scanf (1.0.0) - selenium-webdriver (3.142.7) - childprocess (>= 0.5, < 4.0) - rubyzip (>= 1.2.2) - shacl (0.1.1) - json-ld (~> 3.1, >= 3.1.7) - rdf (~> 3.1, >= 3.1.8) - sparql (~> 3.1) - sxp (~> 1.1) - shex (0.6.1) - ebnf (~> 2.0) - json-ld (~> 3.1) - json-ld-preloaded (~> 3.1) - rdf (~> 3.1) - rdf-xsd (~> 3.1) - sparql (~> 3.1) - sxp (~> 1.1) + selenium-webdriver (4.13.1) + rexml (~> 3.2, >= 3.2.5) + rubyzip (>= 1.2.2, < 3.0) + websocket (~> 1.0) + shacl (0.4.0) + json-ld (~> 3.3) + rdf (~> 3.3) + sparql (~> 3.3) + sxp (~> 1.2) + shex (0.8.0) + ebnf (~> 2.4) + htmlentities (~> 4.3) + json-ld (~> 3.3) + json-ld-preloaded (~> 3.3) + rdf (~> 3.3) + rdf-xsd (~> 3.3) + sparql (~> 3.3) + sxp (~> 1.3) simplecov (0.16.1) docile (~> 1.1) json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - sparql (3.1.7) - builder (~> 3.2) - ebnf (~> 2.1) - rdf (~> 3.1, >= 3.1.14) - rdf-aggregate-repo (~> 3.1) - rdf-xsd (~> 3.1) - sparql-client (~> 3.1, >= 3.1.2) - sxp (~> 1.1) - sparql-client (3.1.2) - net-http-persistent (~> 4.0, >= 4.0.1) - rdf (~> 3.1) + sparql (3.3.0) + builder (~> 3.2, >= 3.2.4) + ebnf (~> 2.4) + logger (~> 1.5) + rdf (~> 3.3) + rdf-aggregate-repo (~> 3.3) + rdf-xsd (~> 3.3) + sparql-client (~> 3.3) + sxp (~> 1.3) + sparql-client (3.3.0) + net-http-persistent (~> 4.0, >= 4.0.2) + rdf (~> 3.3) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) @@ -480,106 +523,112 @@ GEM sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) - sprockets-rails (3.2.2) - actionpack (>= 4.0) - activesupport (>= 4.0) + sprockets-rails (3.4.2) + actionpack (>= 5.2) + activesupport (>= 5.2) sprockets (>= 3.0.0) - sqlite3 (1.4.2) + sqlite3 (1.6.6-x86_64-darwin) + stringio (3.0.8) swagger-docs (0.2.9) activesupport (>= 3) rails (>= 3) - sxp (1.1.0) - rdf (~> 3.1) + sxp (1.3.0) + matrix (~> 0.4) + rdf (~> 3.3) sync (0.5.0) - temple (0.8.2) + temple (0.10.3) term-ansicolor (1.7.1) tins (~> 1.0) - thor (1.1.0) - thread_safe (0.3.6) - tilt (2.0.10) - tins (1.29.1) + thor (1.2.2) + tilt (2.3.0) + timeout (0.4.0) + tins (1.32.1) sync turbolinks (5.2.1) turbolinks-source (~> 5.2) turbolinks-source (5.2.0) - tzinfo (1.2.9) - thread_safe (~> 0.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) uglifier (4.2.0) execjs (>= 0.3.0, < 3) - unicode-display_width (1.7.0) - unicode-types (1.6.0) + unicode-display_width (1.8.0) + unicode-types (1.9.0) useragent (0.16.10) - web-console (3.7.0) - actionview (>= 5.0) - activemodel (>= 5.0) + web-console (4.2.1) + actionview (>= 6.0.0) + activemodel (>= 6.0.0) bindex (>= 0.4.0) - railties (>= 5.0) - webdrivers (4.6.0) + railties (>= 6.0.0) + webdrivers (4.7.0) nokogiri (~> 1.6) rubyzip (>= 1.3.0) - selenium-webdriver (>= 3.0, < 4.0) - webmock (3.14.0) + selenium-webdriver (> 3.141, < 5.0) + webmock (3.19.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - websocket-driver (0.7.5) + websocket (1.2.10) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) xpath (3.2.0) nokogiri (~> 1.8) - xray-rails (0.3.2) - rails (>= 3.1.0) + yaml-ld (0.0.2) + json-ld (~> 3.2, >= 3.2.3) + psych (>= 3.3) + rdf (~> 3.2, >= 3.2.9) + rdf-xsd (~> 3.2) + zeitwerk (2.6.12) PLATFORMS - x86_64-darwin-18 - x86_64-linux + x86_64-darwin-23 DEPENDENCIES better_errors binding_of_caller bixby (~> 3.0) - bootsnap (>= 1.1.0) + bootsnap byebug capybara (>= 2.15) capybara-screenshot (~> 1.0) chromedriver-helper - coffee-rails (~> 4.2) + coffee-rails coveralls dotenv-rails factory_bot (~> 4.4) factory_bot_rails (~> 4.4) faker - jbuilder (~> 2.5) + jbuilder linkeddata - listen (>= 3.0.5, < 3.2) + listen lograge mysql2 - puma (~> 5.6.4) - qa (~> 5.5) - qa_server (~> 7.5) - rails (~> 5.2.5) + puma + qa (~> 5.10) + qa_server! + rails rails-controller-testing rspec-activemodel-mocks (~> 1.0) rspec-its (~> 1.1) rspec-rails (~> 3.1) rspec_junit_formatter rubocop-checkstyle_formatter - sass-rails (~> 5.0) + sass-rails selenium-webdriver spring spring-watcher-listen (~> 2.0.0) sqlite3 swagger-docs - turbolinks (~> 5) + turbolinks tzinfo-data - uglifier (>= 1.3.0) + uglifier web-console (>= 3.3.0) webdrivers (~> 4.4) webmock - xray-rails + xray-rails! RUBY VERSION - ruby 2.7.3p183 + ruby 3.1.2p20 BUNDLED WITH - 2.2.17 + 2.3.7 From b69173893de6a89307ca8935eeed988fd05aef66 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Tue, 31 Oct 2023 09:03:21 -0400 Subject: [PATCH 04/22] Extend loc lookup for loc names rdftypes --- lib/qa/authorities/loc.rb | 20 ++++ lib/qa/authorities/loc/generic_authority.rb | 94 +++++++++++++++++ lib/qa/authorities/loc_subauthority.rb | 107 ++++++++++++++++++++ 3 files changed, 221 insertions(+) create mode 100644 lib/qa/authorities/loc.rb create mode 100644 lib/qa/authorities/loc/generic_authority.rb create mode 100644 lib/qa/authorities/loc_subauthority.rb diff --git a/lib/qa/authorities/loc.rb b/lib/qa/authorities/loc.rb new file mode 100644 index 0000000..510edd5 --- /dev/null +++ b/lib/qa/authorities/loc.rb @@ -0,0 +1,20 @@ +# Overrides loc authority from qa gem to handle rdftypes for subauthorities + +require 'uri' + +module Qa::Authorities + module Loc + extend AuthorityWithSubAuthority + + require 'qa/authorities/loc/generic_authority' + def self.subauthority_for(subauthority) + validate_subauthority!(subauthority) + GenericAuthority.new(subauthority) + end + + extend LocSubauthority + def self.subauthorities + authorities + vocabularies + datatypes + preservation + rdftypes + end + end +end diff --git a/lib/qa/authorities/loc/generic_authority.rb b/lib/qa/authorities/loc/generic_authority.rb new file mode 100644 index 0000000..d138c1f --- /dev/null +++ b/lib/qa/authorities/loc/generic_authority.rb @@ -0,0 +1,94 @@ +# Overrides loc authority from qa gem to customize search output and handle +# rdftypes for subauthorities + +module Qa::Authorities + class Loc::GenericAuthority < Base + + attr_reader :subauthority + def initialize(subauthority) + @subauthority = subauthority + end + + include WebServiceBase + + def response(url) + uri = URI(url) + conn = Faraday.new "#{uri.scheme}://#{uri.host}" + conn.options.params_encoder = Faraday::FlatParamsEncoder + conn.get do |req| + req.headers['Accept'] = 'application/json' + req.url uri.path + req.params = Rack::Utils.parse_query(uri.query) + end + end + + def search(q, tc) + @raw_response = json(build_query_url(q, tc)) + parse_authority_response + end + + def build_query_url(q, tc) + escaped_query = ERB::Util.url_encode(q) + authority = Loc.get_authority(subauthority) + rdftype = Loc.get_rdftype(subauthority) + authority_fragment = Loc.get_url_for_authority(authority) + ERB::Util.url_encode(authority) + rdftype_fragment = "&q=rdftype:#{ERB::Util.url_encode(rdftype)}" if rdftype + "https://id.loc.gov/search/?q=#{escaped_query}&q=#{authority_fragment}#{rdftype_fragment}&format=json" + end + + def find(id) + json(find_url(id)) + end + + def find_url(id) + "https://id.loc.gov/authorities/#{@subauthority}/#{id}.json" + end + + private + + # Reformats the data received from the LOC service + def parse_authority_response + @raw_response.select { |response| response[0] == "atom:entry" }.map do |response| + loc_response_to_qa(response_to_struct(response)) + end + end + + # Converts most of the atom data into an OpenStruct object. + # + # Note that this is a pretty naive conversion. There should probably just + # be a class that properly translates and stores the various pieces of + # data, especially if this logic could be useful in other auth lookups. + def response_to_struct(response) + contents = response.each_with_object({}) do |result_parts, result| + next unless result_parts[0] + key = result_parts[0].sub('atom:', '').sub('dcterms:', '') + info = result_parts[1] + val = result_parts[2] + + case key + when 'title', 'id', 'name', 'updated', 'created' + result[key] = val + when 'link' + result["links"] ||= [] + result["links"] << { "type" => info["type"], "href" => info["href"] } + end + end + + OpenStruct.new(contents) + end + + # Simple conversion from LoC-based struct to QA hash + def loc_response_to_qa(data) + response = { + "id" => data.id || data.title, + "label" => data.title + } + + if data.links.present? + response["uri"] = (data.links.find { |l| l["type"].nil? } || data.links[0])["href"] + end + + response + end + end +end diff --git a/lib/qa/authorities/loc_subauthority.rb b/lib/qa/authorities/loc_subauthority.rb new file mode 100644 index 0000000..fcf0c96 --- /dev/null +++ b/lib/qa/authorities/loc_subauthority.rb @@ -0,0 +1,107 @@ +# Overrides loc authority from qa gem to handle rdftypes for subauthorities + +module Qa::Authorities::LocSubauthority + def get_url_for_authority(authority) + if authorities.include?(authority) then authority_base_url + elsif vocabularies.include?(authority) then vocab_base_url + elsif datatypes.include?(authority) then datatype_base_url + elsif preservation.include?(authority) then vocab_preservation_base_url + end + end + + def get_authority(authority) + rdftypes.include?(authority) ? authority.split('_')[0] : authority + end + + def get_rdftype(authority) + rdftypes.include?(authority) ? authority.split('_')[-1] : nil + end + + def authorities + [ + "subjects", + "names", + "classification", + "childrensSubjects", + "genreForms", + "performanceMediums" + ] + end + + def vocabularies # rubocop:disable Metrics/MethodLength + [ + "graphicMaterials", + "organizations", + "relators", + "countries", + "ethnographicTerms", + "geographicAreas", + "languages", + "iso639-1", + "iso639-2", + "iso639-5", + "preservation", + "actionsGranted", + "agentType" + ] + end + + def datatypes + ["edtf"] + end + + def preservation # rubocop:disable Metrics/MethodLength + [ + "contentLocationType", + "copyrightStatus", + "cryptographicHashFunctions", + "environmentCharacteristic", + "environmentPurpose", + "eventRelatedAgentRole", + "eventRelatedObjectRole", + "eventType", + "formatRegistryRole", + "hardwareType", + "inhibitorTarget", + "inhibitorType", + "objectCategory", + "preservationLevelRole", + "relationshipSubType", + "relationshipType", + "rightsBasis", + "rightsRelatedAgentRole", + "signatureEncoding", + "signatureMethod", + "softwareType", + "storageMedium" + ] + end + + def rdftypes + [ + "names_ConferenceName", + "names_CorporateName", + "names_FamilyName", + "names_Geographic", + "names_PersonalName" + ] + end + + private + + def vocab_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2F" + end + + def authority_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fauthorities%2F" + end + + def datatype_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fdatatypes%2F" + end + + def vocab_preservation_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2Fpreservation%2F" + end +end From be73b2143746e0ee40564ee3a780f8df25734aeb Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Tue, 31 Oct 2023 10:24:48 -0400 Subject: [PATCH 05/22] Add subauthority support for more loc vocabularies and authorities --- lib/qa/authorities/loc_subauthority.rb | 72 ++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/lib/qa/authorities/loc_subauthority.rb b/lib/qa/authorities/loc_subauthority.rb index fcf0c96..a5d0955 100644 --- a/lib/qa/authorities/loc_subauthority.rb +++ b/lib/qa/authorities/loc_subauthority.rb @@ -19,30 +19,84 @@ def get_rdftype(authority) def authorities [ - "subjects", - "names", "classification", "childrensSubjects", + "demographicTerms", "genreForms", - "performanceMediums" + "names", + "performanceMediums", + "subjects", ] end def vocabularies # rubocop:disable Metrics/MethodLength [ - "graphicMaterials", - "organizations", - "relators", + "actionsGranted", + "agentType", + "bookformat", + "carriers", + "classSchemes", + "contentTypes", "countries", + "descriptionConventions", "ethnographicTerms", + "frequencies", "geographicAreas", - "languages", + "graphicMaterials", "iso639-1", "iso639-2", "iso639-5", + "issuance", + "languages", + "marcauthen", + "maspect", + "maudience", + "mbroadstd", + "mcapturestorage", + "mcolor", + "mediaTypes", + "mencformat", + "menclvl", + "mfiletype", + "mfont", + "mgeneration", + "mgovtpubtype", + "mgroove", + "millus", + "mlayout", + "mmaterial", + "mmusicformat", + "mmusnotation", + "mnotetype", + "mplayback", + "mplayspeed", + "mpolarity", + "mpresformat", + "mproduction", + "mprojection", + "mrecmedium", + "mrectype", + "mreductionratio", + "mregencoding", + "mrelief", + "mscale", + "mscript", + "mserialpubtype", + "msoundcontent", + "mspecplayback", + "mstatus", + "msupplcont", + "mtactile", + "mtapeconfig", + "mtechnique", + "mvidformat", + "organizations", "preservation", - "actionsGranted", - "agentType" + "rbmscv", + "rbmsrel", + "relationship", + "relators", + "resourceComponents", ] end From c6f83466f5d6d80740449b5cdf939174d23587eb Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Tue, 31 Oct 2023 10:52:23 -0400 Subject: [PATCH 06/22] Add id uniqueness filter to loc direct lookup output --- lib/qa/authorities/loc/generic_authority.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/qa/authorities/loc/generic_authority.rb b/lib/qa/authorities/loc/generic_authority.rb index d138c1f..a1d669e 100644 --- a/lib/qa/authorities/loc/generic_authority.rb +++ b/lib/qa/authorities/loc/generic_authority.rb @@ -47,10 +47,11 @@ def find_url(id) private # Reformats the data received from the LOC service + # Filters for id uniqueness def parse_authority_response @raw_response.select { |response| response[0] == "atom:entry" }.map do |response| loc_response_to_qa(response_to_struct(response)) - end + end.uniq { |result| result["id"] } end # Converts most of the atom data into an OpenStruct object. From 5e1a615df4e55a5bc1d8cb10ec0f1abcdfc1917c Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Tue, 31 Oct 2023 15:38:57 -0400 Subject: [PATCH 07/22] Fix rubocop warnings --- lib/qa/authorities/loc/generic_authority.rb | 82 ++++++++++----------- lib/qa/authorities/loc_subauthority.rb | 30 ++++---- 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/lib/qa/authorities/loc/generic_authority.rb b/lib/qa/authorities/loc/generic_authority.rb index a1d669e..fc894a0 100644 --- a/lib/qa/authorities/loc/generic_authority.rb +++ b/lib/qa/authorities/loc/generic_authority.rb @@ -3,7 +3,6 @@ module Qa::Authorities class Loc::GenericAuthority < Base - attr_reader :subauthority def initialize(subauthority) @subauthority = subauthority @@ -22,12 +21,12 @@ def response(url) end end - def search(q, tc) - @raw_response = json(build_query_url(q, tc)) + def search(q) + @raw_response = json(build_query_url(q)) parse_authority_response end - def build_query_url(q, tc) + def build_query_url(q) escaped_query = ERB::Util.url_encode(q) authority = Loc.get_authority(subauthority) rdftype = Loc.get_rdftype(subauthority) @@ -46,50 +45,49 @@ def find_url(id) private - # Reformats the data received from the LOC service - # Filters for id uniqueness - def parse_authority_response - @raw_response.select { |response| response[0] == "atom:entry" }.map do |response| - loc_response_to_qa(response_to_struct(response)) - end.uniq { |result| result["id"] } + # Reformats the data received from the LOC service + # Filters for id uniqueness + def parse_authority_response + results = @raw_response.select { |response| response[0] == "atom:entry" }.map do |response| + loc_response_to_qa(response_to_struct(response)) end + results.uniq { |result| result["id"] } + end - # Converts most of the atom data into an OpenStruct object. - # - # Note that this is a pretty naive conversion. There should probably just - # be a class that properly translates and stores the various pieces of - # data, especially if this logic could be useful in other auth lookups. - def response_to_struct(response) - contents = response.each_with_object({}) do |result_parts, result| - next unless result_parts[0] - key = result_parts[0].sub('atom:', '').sub('dcterms:', '') - info = result_parts[1] - val = result_parts[2] - - case key - when 'title', 'id', 'name', 'updated', 'created' - result[key] = val - when 'link' - result["links"] ||= [] - result["links"] << { "type" => info["type"], "href" => info["href"] } - end - end + # Converts most of the atom data into an OpenStruct object. + # + # Note that this is a pretty naive conversion. There should probably just + # be a class that properly translates and stores the various pieces of + # data, especially if this logic could be useful in other auth lookups. + def response_to_struct(response) + contents = response.each_with_object({}) do |result_parts, result| + next unless result_parts[0] + key = result_parts[0].sub('atom:', '').sub('dcterms:', '') + info = result_parts[1] + val = result_parts[2] - OpenStruct.new(contents) + case key + when 'title', 'id', 'name', 'updated', 'created' + result[key] = val + when 'link' + result["links"] ||= [] + result["links"] << { "type" => info["type"], "href" => info["href"] } + end end - # Simple conversion from LoC-based struct to QA hash - def loc_response_to_qa(data) - response = { - "id" => data.id || data.title, - "label" => data.title - } + OpenStruct.new(contents) + end + + # Simple conversion from LoC-based struct to QA hash + def loc_response_to_qa(data) + response = { + "id" => data.id || data.title, + "label" => data.title + } - if data.links.present? - response["uri"] = (data.links.find { |l| l["type"].nil? } || data.links[0])["href"] - end + response["uri"] = (data.links.find { |l| l["type"].nil? } || data.links[0])["href"] if data.links.present? - response - end + response + end end end diff --git a/lib/qa/authorities/loc_subauthority.rb b/lib/qa/authorities/loc_subauthority.rb index a5d0955..76c3411 100644 --- a/lib/qa/authorities/loc_subauthority.rb +++ b/lib/qa/authorities/loc_subauthority.rb @@ -1,6 +1,6 @@ # Overrides loc authority from qa gem to handle rdftypes for subauthorities -module Qa::Authorities::LocSubauthority +module Qa::Authorities::LocSubauthority # rubocop:disable Metrics/ModuleLength def get_url_for_authority(authority) if authorities.include?(authority) then authority_base_url elsif vocabularies.include?(authority) then vocab_base_url @@ -25,7 +25,7 @@ def authorities "genreForms", "names", "performanceMediums", - "subjects", + "subjects" ] end @@ -96,7 +96,7 @@ def vocabularies # rubocop:disable Metrics/MethodLength "rbmsrel", "relationship", "relators", - "resourceComponents", + "resourceComponents" ] end @@ -143,19 +143,19 @@ def rdftypes private - def vocab_base_url - "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2F" - end + def vocab_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2F" + end - def authority_base_url - "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fauthorities%2F" - end + def authority_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fauthorities%2F" + end - def datatype_base_url - "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fdatatypes%2F" - end + def datatype_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fdatatypes%2F" + end - def vocab_preservation_base_url - "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2Fpreservation%2F" - end + def vocab_preservation_base_url + "cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2Fpreservation%2F" + end end From 8eff316ae9886724fb2dead9292dc0774cf3392a Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Wed, 1 Nov 2023 09:31:44 -0400 Subject: [PATCH 08/22] Don't ignore rubocop configuration in docker container --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 5cf72d5..38a3e89 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,7 +10,6 @@ /.git /.gitignore /.rspec -/.rubocop* /.qa_server_app /coverage/* From c7c092e169b2114d8e7b885df30abceaf69ffb9d Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Wed, 1 Nov 2023 14:16:35 -0400 Subject: [PATCH 09/22] Revert changes for local copy of qa_server --- Dockerfile | 8 +------- Gemfile | 13 ++----------- Gemfile.lock | 53 +++++++++++++++++++++++++++------------------------- 3 files changed, 31 insertions(+), 43 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee030fd..8873e6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,13 +30,7 @@ ENV RAILS_ROOT="/app/ld4p/qa_server-webapp" COPY Gemfile Gemfile.lock ./ RUN gem update --system - -## TEMPORARY TO UPDATE RAILS -## Remove this after updating qa_server with new rails version, -## and uncomment RUN bundle install -COPY qa_server /tmp/qa_server -RUN bundle config --local path /tmp/qa_server && bundle install -#RUN bundle install +RUN bundle install COPY . . RUN bundle exec rake assets:precompile diff --git a/Gemfile b/Gemfile index cb1f1ec..729be95 100644 --- a/Gemfile +++ b/Gemfile @@ -8,7 +8,7 @@ ruby '3.1.2' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' gem 'rails', '~> 7.0.8' # Use sqlite3 as the database for Active Record -gem 'sqlite3' +gem 'sqlite3','~> 1.4.4' # Use Puma as the app server gem 'puma', '>= 5.6.4', '~>6.4.0' # Use SCSS for stylesheets @@ -65,16 +65,7 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] ## Gems manually added to for qa and qa_server engines # Required gems for QA and linked data access -# gem 'qa_server', '~> 7.5' - -# TEMPORARY TO UPDATE RAILS -# Remove these two qa_server entries after updating qa_server with new rails version. -# First for building in Docker: -gem 'qa_server', path: '/tmp/qa_server' -# Next line for building as a standard Rails project sans Docker. -# This assumes that the remote qa_server is a subdirectory of the main project (which makes -# things so much easier when we are using Docker). -# gem 'qa_server', path: 'qa_server' +gem 'qa_server', git: "https://github.com/LD4P/qa_server.git", branch: 'update_rails' gem 'qa', '~> 5.10' gem 'linkeddata' gem 'psych', '~> 5.1' diff --git a/Gemfile.lock b/Gemfile.lock index fcb26ce..d81c2c6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,14 @@ +GIT + remote: https://github.com/LD4P/qa_server.git + revision: 6476f63031bb0ecae498bb421e8ce2c8a750aa69 + branch: update_rails + specs: + qa_server (8.0.0) + gruff + rails (~> 7.0) + sass-rails (~> 5.0) + useragent + GIT remote: https://github.com/brentd/xray-rails.git revision: f121814718c9907b20058dc9357b80a53afab821 @@ -6,15 +17,6 @@ GIT xray-rails (0.3.2) rails (>= 3.1.0) -PATH - remote: qa_server - specs: - qa_server (7.9.2) - gruff - rails (~> 7.0) - sass-rails (~> 5.0) - useragent - GEM remote: https://rubygems.org/ specs: @@ -257,7 +259,7 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.21.3) + loofah (2.21.4) crass (~> 1.0.2) nokogiri (>= 1.12.0) mail (2.8.1) @@ -275,7 +277,7 @@ GEM mysql2 (0.5.5) net-http-persistent (4.0.2) connection_pool (~> 2.2) - net-imap (0.4.0) + net-imap (0.4.3) date net-protocol net-pop (0.1.2) @@ -307,7 +309,7 @@ GEM nokogiri (~> 1.6) rails (>= 5.0, < 7.1) rdf - racc (1.7.1) + racc (1.7.2) rack (2.2.8) rack-test (2.1.0) rack (>= 1.3) @@ -344,7 +346,7 @@ GEM thor (~> 1.0) zeitwerk (~> 2.5) rainbow (3.1.1) - rake (13.0.6) + rake (13.1.0) rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) @@ -527,7 +529,7 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - sqlite3 (1.6.6-x86_64-darwin) + sqlite3 (1.4.4) stringio (3.0.8) swagger-docs (0.2.9) activesupport (>= 3) @@ -539,7 +541,7 @@ GEM temple (0.10.3) term-ansicolor (1.7.1) tins (~> 1.0) - thor (1.2.2) + thor (1.3.0) tilt (2.3.0) timeout (0.4.0) tins (1.32.1) @@ -587,41 +589,42 @@ DEPENDENCIES better_errors binding_of_caller bixby (~> 3.0) - bootsnap + bootsnap (~> 1.16) byebug capybara (>= 2.15) capybara-screenshot (~> 1.0) chromedriver-helper - coffee-rails + coffee-rails (~> 5.0.0) coveralls dotenv-rails factory_bot (~> 4.4) factory_bot_rails (~> 4.4) faker - jbuilder + jbuilder (~> 2.10) linkeddata - listen + listen (>= 3.0.5, < 3.9) lograge mysql2 - puma + psych (~> 5.1) + puma (~> 6.4.0, >= 5.6.4) qa (~> 5.10) qa_server! - rails + rails (~> 7.0.8) rails-controller-testing rspec-activemodel-mocks (~> 1.0) rspec-its (~> 1.1) rspec-rails (~> 3.1) rspec_junit_formatter rubocop-checkstyle_formatter - sass-rails + sass-rails (~> 5.0) selenium-webdriver spring spring-watcher-listen (~> 2.0.0) - sqlite3 + sqlite3 (~> 1.4.4) swagger-docs - turbolinks + turbolinks (~> 5) tzinfo-data - uglifier + uglifier (>= 1.3.0) web-console (>= 3.3.0) webdrivers (~> 4.4) webmock From f7f66080c7ae90c041acef1adbb9708979dd99d5 Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Wed, 1 Nov 2023 14:30:52 -0400 Subject: [PATCH 10/22] Add gcompat for architecture independence; update Bundler --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8873e6a..ac174e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ ARG RUBY_VERSION=3.1.2 FROM ruby:$RUBY_VERSION-alpine +ARG BUNDLER_VERSION=2.3.7 ## Install dependencies: ## - build-base: To ensure certain gems can be compiled @@ -9,6 +10,7 @@ FROM ruby:$RUBY_VERSION-alpine ## - tzdata: add time zone support ## - mariadb-dev: To allow use of MySQL2 gem ## - imagemagick: for image processing +## - gcompat: to avoid architecture-specific incompatibitilies RUN apk add --update --no-cache \ bash \ build-base \ @@ -17,12 +19,13 @@ RUN apk add --update --no-cache \ sqlite-dev \ tzdata \ mariadb-dev \ - imagemagick6-dev imagemagick6-libs + imagemagick6-dev imagemagick6-libs \ + gcompat WORKDIR /app/ld4p/qa_server-webapp -RUN gem install bundler:2.1.4 +RUN gem install bundler:${BUNDLER_VERSION} ENV PATH="/app/ld4p/qa_server-webapp:$PATH" ENV RAILS_ROOT="/app/ld4p/qa_server-webapp" From f2941ad5a41f1b3b44660ae69bdcf39392be98fa Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Wed, 1 Nov 2023 15:00:53 -0400 Subject: [PATCH 11/22] Update bixby and appease rubocop --- Gemfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 729be95..9918240 100644 --- a/Gemfile +++ b/Gemfile @@ -8,7 +8,7 @@ ruby '3.1.2' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' gem 'rails', '~> 7.0.8' # Use sqlite3 as the database for Active Record -gem 'sqlite3','~> 1.4.4' +gem 'sqlite3', '~> 1.4.4' # Use Puma as the app server gem 'puma', '>= 5.6.4', '~>6.4.0' # Use SCSS for stylesheets @@ -99,7 +99,7 @@ end group :development do gem 'better_errors' # add command line in browser when errors gem 'binding_of_caller' # deeper stack trace used by better errors - gem 'bixby', '~> 3.0' # style guide enforcement with rubocop + gem 'bixby', '~> 5.0' # style guide enforcement with rubocop gem 'rubocop-checkstyle_formatter', require: false end From 4daf3126e4eb64d9f67aebbdf975695ee7979152 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Wed, 1 Nov 2023 15:24:51 -0400 Subject: [PATCH 12/22] Update bixby dependencies in Gemfile.lock --- Gemfile.lock | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index d81c2c6..e4a622f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -101,9 +101,9 @@ GEM bindex (0.8.1) binding_of_caller (1.0.0) debug_inspector (>= 0.0.1) - bixby (3.0.2) - rubocop (= 0.85.1) - rubocop-ast (~> 0.3.0) + bixby (5.0.2) + rubocop (= 1.28.2) + rubocop-ast rubocop-performance rubocop-rails rubocop-rspec @@ -287,6 +287,8 @@ GEM net-smtp (0.4.0) net-protocol nio4r (2.5.9) + nokogiri (1.15.4-arm64-darwin) + racc (~> 1.4) nokogiri (1.15.4-x86_64-darwin) racc (~> 1.4) parallel (1.23.0) @@ -414,7 +416,7 @@ GEM rdf-xsd (3.3.0) rdf (~> 3.3) rexml (~> 3.2) - regexp_parser (2.8.1) + regexp_parser (2.8.2) request_store (1.5.1) rack (>= 1.4) rexml (3.2.6) @@ -447,27 +449,28 @@ GEM rspec-support (3.9.4) rspec_junit_formatter (0.6.0) rspec-core (>= 2, < 4, != 2.12.0) - rubocop (0.85.1) + rubocop (1.28.2) parallel (~> 1.10) - parser (>= 2.7.0.1) + parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.7) + regexp_parser (>= 1.8, < 3.0) rexml - rubocop-ast (>= 0.0.3) + rubocop-ast (>= 1.17.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (0.3.0) - parser (>= 2.7.1.4) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) rubocop-checkstyle_formatter (0.4.0) rubocop (>= 0.35.1) - rubocop-performance (1.7.1) - rubocop (>= 0.82.0) - rubocop-rails (2.6.0) + rubocop-performance (1.19.1) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.15.2) activesupport (>= 4.2.0) rack (>= 1.1) - rubocop (>= 0.82.0) - rubocop-rspec (1.41.0) - rubocop (>= 0.68.1) + rubocop (>= 1.7.0, < 2.0) + rubocop-rspec (2.11.1) + rubocop (~> 1.19) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) @@ -553,7 +556,7 @@ GEM concurrent-ruby (~> 1.0) uglifier (4.2.0) execjs (>= 0.3.0, < 3) - unicode-display_width (1.8.0) + unicode-display_width (2.5.0) unicode-types (1.9.0) useragent (0.16.10) web-console (4.2.1) @@ -583,12 +586,13 @@ GEM zeitwerk (2.6.12) PLATFORMS + arm64-darwin-22 x86_64-darwin-23 DEPENDENCIES better_errors binding_of_caller - bixby (~> 3.0) + bixby (~> 5.0) bootsnap (~> 1.16) byebug capybara (>= 2.15) From 43cfdfef1e36ab903fb9dd2d19b02e7f69575e14 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Wed, 1 Nov 2023 15:26:38 -0400 Subject: [PATCH 13/22] Disable Lint/MissingSuper on classes inheriting from qa base parent --- app/authorities/qa/authorities/wikidata/generic_authority.rb | 2 +- lib/qa/authorities/loc/generic_authority.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/authorities/qa/authorities/wikidata/generic_authority.rb b/app/authorities/qa/authorities/wikidata/generic_authority.rb index e1cc1e3..ccf1d3e 100644 --- a/app/authorities/qa/authorities/wikidata/generic_authority.rb +++ b/app/authorities/qa/authorities/wikidata/generic_authority.rb @@ -4,7 +4,7 @@ module Qa::Authorities # http://www.oclc.org/developer/develop/web-services/fast-api/assign-fast.en.html class Wikidata::GenericAuthority < Base attr_reader :subauthority - def initialize(subauthority) + def initialize(subauthority) # rubocop:disable Lint/MissingSuper @subauthority = subauthority end diff --git a/lib/qa/authorities/loc/generic_authority.rb b/lib/qa/authorities/loc/generic_authority.rb index fc894a0..4ce4606 100644 --- a/lib/qa/authorities/loc/generic_authority.rb +++ b/lib/qa/authorities/loc/generic_authority.rb @@ -4,7 +4,7 @@ module Qa::Authorities class Loc::GenericAuthority < Base attr_reader :subauthority - def initialize(subauthority) + def initialize(subauthority) # rubocop:disable Lint/MissingSuper @subauthority = subauthority end From d9f6f2a9c5e85a9be299117cf7c878d6eeae5ad2 Mon Sep 17 00:00:00 2001 From: Christina Cortland Date: Wed, 1 Nov 2023 15:26:51 -0400 Subject: [PATCH 14/22] Update .env.example --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 905e568..1cda9b5 100644 --- a/.env.example +++ b/.env.example @@ -58,10 +58,10 @@ MYSQL_ROOT_PASSWORD=CHANGEME ## -------------------- PREFERRED_TIME_ZONE_NAME=Eastern Time (US & Canada) DISPLAY_HISTORICAL_DATA=true -HISTORICAL_DATA_DEFAULT_TIME_PERIOD=:month +HISTORICAL_DATA_DEFAULT_TIME_PERIOD=month DISPLAY_PERFORMANCE_GRAPH=false DISPLAY_PERFORMANCE_DATATABLE=false -PERFORMANCE_DATA_DEFAULT_TIME_PERIOD=:month +PERFORMANCE_DATA_DEFAULT_TIME_PERIOD=month SUPPRESS_PERFORMANCE_LOGGING=true MONITOR_STATUS_LOGGING_ENABLED=false From f3f4077f59ae68e529bb02d6692da36d9577dba2 Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Fri, 10 Nov 2023 15:26:15 -0500 Subject: [PATCH 15/22] Update reference to qa_server --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 9918240..42352a7 100644 --- a/Gemfile +++ b/Gemfile @@ -65,7 +65,7 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] ## Gems manually added to for qa and qa_server engines # Required gems for QA and linked data access -gem 'qa_server', git: "https://github.com/LD4P/qa_server.git", branch: 'update_rails' +gem 'qa_server', git: 'https://github.com/LD4P/qa_server.git', tag: 'v8.0' gem 'qa', '~> 5.10' gem 'linkeddata' gem 'psych', '~> 5.1' From e2b4cd14bf0312f32635999eb823dab9f0f330d0 Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Fri, 10 Nov 2023 15:27:52 -0500 Subject: [PATCH 16/22] Update reference to qa_server --- Gemfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gemfile b/Gemfile index 42352a7..d5c05d8 100644 --- a/Gemfile +++ b/Gemfile @@ -65,6 +65,8 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] ## Gems manually added to for qa and qa_server engines # Required gems for QA and linked data access +# NOTE: We need to update the published RubyGems version of qa_server with the 8.0 release. Until +# then, the next entry is set to use the GitHub repo and tag instead. gem 'qa_server', git: 'https://github.com/LD4P/qa_server.git', tag: 'v8.0' gem 'qa', '~> 5.10' gem 'linkeddata' From f816027db9d237ec62424d3813bc50715a162138 Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Fri, 10 Nov 2023 15:39:06 -0500 Subject: [PATCH 17/22] Update reference to qa_server --- Gemfile.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index e4a622f..88dd3b7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/LD4P/qa_server.git - revision: 6476f63031bb0ecae498bb421e8ce2c8a750aa69 - branch: update_rails + revision: af1f6509919b07b188a4714937aa33456614bf57 + tag: v8.0 specs: qa_server (8.0.0) gruff @@ -144,7 +144,7 @@ GEM crack (0.4.5) rexml crass (1.0.6) - date (3.3.3) + date (3.3.4) debug_inspector (1.1.0) deprecation (1.1.0) activesupport @@ -277,12 +277,12 @@ GEM mysql2 (0.5.5) net-http-persistent (4.0.2) connection_pool (~> 2.2) - net-imap (0.4.3) + net-imap (0.4.4) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout net-smtp (0.4.0) net-protocol @@ -311,7 +311,7 @@ GEM nokogiri (~> 1.6) rails (>= 5.0, < 7.1) rdf - racc (1.7.2) + racc (1.7.3) rack (2.2.8) rack-test (2.1.0) rack (>= 1.3) @@ -546,7 +546,7 @@ GEM tins (~> 1.0) thor (1.3.0) tilt (2.3.0) - timeout (0.4.0) + timeout (0.4.1) tins (1.32.1) sync turbolinks (5.2.1) From cfd64e6ffc98a8c5792d434867278a27bcd5a3ed Mon Sep 17 00:00:00 2001 From: Matthew Connolly Date: Fri, 10 Nov 2023 16:14:56 -0500 Subject: [PATCH 18/22] Apply rails app:update --- bin/rails | 6 +- bin/rake | 4 +- bin/setup | 31 ++-- config/boot.rb | 4 +- config/environment.rb | 2 +- config/environments/development.rb | 50 +++--- config/environments/production.rb | 53 ++++--- config/environments/test.rb | 40 +++-- .../initializers/content_security_policy.rb | 25 +++ .../initializers/filter_parameter_logging.rb | 8 +- config/initializers/inflections.rb | 8 +- .../new_framework_defaults_7_0.rb | 143 ++++++++++++++++++ config/initializers/permissions_policy.rb | 11 ++ config/storage.yml | 34 +++++ ..._to_active_storage_blobs.active_storage.rb | 22 +++ ..._storage_variant_records.active_storage.rb | 27 ++++ ...e_storage_blobs_checksum.active_storage.rb | 8 + 17 files changed, 383 insertions(+), 93 deletions(-) create mode 100644 config/initializers/content_security_policy.rb create mode 100644 config/initializers/new_framework_defaults_7_0.rb create mode 100644 config/initializers/permissions_policy.rb create mode 100644 config/storage.yml create mode 100644 db/migrate/20231110211250_add_service_name_to_active_storage_blobs.active_storage.rb create mode 100644 db/migrate/20231110211251_create_active_storage_variant_records.active_storage.rb create mode 100644 db/migrate/20231110211252_remove_not_null_on_active_storage_blobs_checksum.active_storage.rb diff --git a/bin/rails b/bin/rails index 0739660..efc0377 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,4 @@ #!/usr/bin/env ruby -APP_PATH = File.expand_path('../config/application', __dir__) -require_relative '../config/boot' -require 'rails/commands' +APP_PATH = File.expand_path("../config/application", __dir__) +require_relative "../config/boot" +require "rails/commands" diff --git a/bin/rake b/bin/rake index 1724048..4fbf10b 100755 --- a/bin/rake +++ b/bin/rake @@ -1,4 +1,4 @@ #!/usr/bin/env ruby -require_relative '../config/boot' -require 'rake' +require_relative "../config/boot" +require "rake" Rake.application.run diff --git a/bin/setup b/bin/setup index 78c4e86..ec47b79 100755 --- a/bin/setup +++ b/bin/setup @@ -1,38 +1,33 @@ #!/usr/bin/env ruby -require 'pathname' -require 'fileutils' -include FileUtils +require "fileutils" # path to your application root. -APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) +APP_ROOT = File.expand_path("..", __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") end -chdir APP_ROOT do - # This script is a starting point to setup your application. +FileUtils.chdir APP_ROOT do + # This script is a way to set up or update your development environment automatically. + # This script is idempotent, so that you can run it at any time and get an expectable outcome. # Add necessary setup steps to this file. - puts '== Installing dependencies ==' - system! 'gem install bundler --conservative' - system('bundle check') || system!('bundle install') - - # Install JavaScript dependencies if using Yarn - # system('bin/yarn') - + puts "== Installing dependencies ==" + system! "gem install bundler --conservative" + system("bundle check") || system!("bundle install") # puts "\n== Copying sample files ==" - # unless File.exist?('config/database.yml') - # cp 'config/database.yml.sample', 'config/database.yml' + # unless File.exist?("config/database.yml") + # FileUtils.cp "config/database.yml.sample", "config/database.yml" # end puts "\n== Preparing database ==" - system! 'bin/rails db:setup' + system! "bin/rails db:prepare" puts "\n== Removing old logs and tempfiles ==" - system! 'bin/rails log:clear tmp:clear' + system! "bin/rails log:clear tmp:clear" puts "\n== Restarting application server ==" - system! 'bin/rails restart' + system! "bin/rails restart" end diff --git a/config/boot.rb b/config/boot.rb index 30f5120..2820116 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,3 +1,3 @@ -ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) +ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) -require 'bundler/setup' # Set up gems listed in the Gemfile. +require "bundler/setup" # Set up gems listed in the Gemfile. diff --git a/config/environment.rb b/config/environment.rb index 426333b..cac5315 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,5 +1,5 @@ # Load the Rails application. -require_relative 'application' +require_relative "application" # Initialize the Rails application. Rails.application.initialize! diff --git a/config/environments/development.rb b/config/environments/development.rb index 09c6a74..8500f45 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -1,8 +1,10 @@ +require "active_support/core_ext/integer/time" + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. - # In the development environment your application's code is reloaded on - # every request. This slows down response time but is perfect for development + # In the development environment your application's code is reloaded any time + # it changes. This slows down response time but is perfect for development # since you don't have to restart the web server when you make code changes. config.cache_classes = false @@ -12,14 +14,18 @@ # Show full error reports. config.consider_all_requests_local = true + # Enable server timing + config.server_timing = true + # Enable/disable caching. By default caching is disabled. - if Rails.root.join('tmp', 'caching-dev.txt').exist? - # Run rails dev:cache to toggle caching in a file (used in development). + # Run rails dev:cache to toggle caching. + if Rails.root.join("tmp/caching-dev.txt").exist? config.action_controller.perform_caching = true + config.action_controller.enable_fragment_cache_logging = true config.cache_store = :memory_store config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{2.days.to_i}" + "Cache-Control" => "public, max-age=#{2.days.to_i}" } else config.action_controller.perform_caching = false @@ -27,12 +33,8 @@ config.cache_store = :null_store end - # Use the lowest log level to ensure availability of diagnostic information - # when problems arise. - config.log_level = :info - - # Prepend all log lines with the following tags. - config.log_tags = [:request_id] + # Store uploaded files on the local file system (see config/storage.yml for options). + config.active_storage.service = :local # Don't care if the mailer can't send. config.action_mailer.raise_delivery_errors = false @@ -42,25 +44,27 @@ # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load - # Debug mode disables concatenation and preprocessing of assets. - # This option may cause significant delays in view rendering with a large - # number of complex assets. - config.assets.debug = true + # Highlight code that triggered database queries in logs. + config.active_record.verbose_query_logs = true # Suppress logger output for asset requests. config.assets.quiet = true - # Raises error for missing translations - # raise_on_missing_translations DEPRECATED IN RAILS 7 - # config.action_view.raise_on_missing_translations = true + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true - # Use an evented file watcher to asynchronously detect changes in source code, - # routes, locales, etc. This feature depends on the listen gem. - config.file_watcher = ActiveSupport::EventedFileUpdateChecker + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true - config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging - # config.active_job.queue_adapter = :inline # runs when it is called for debugging + # Uncomment if you wish to allow Action Cable access from any origin. + # config.action_cable.disable_request_forgery_protection = true end diff --git a/config/environments/production.rb b/config/environments/production.rb index 4c810c1..3a44911 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,3 +1,5 @@ +require "active_support/core_ext/integer/time" + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -14,50 +16,51 @@ config.consider_all_requests_local = false config.action_controller.perform_caching = true + # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] + # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true + # Disable serving static files from the `/public` folder by default since # Apache or NGINX already handles this. - config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? + config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present? - # Compress JavaScripts and CSS. - config.assets.js_compressor = :uglifier + # Compress CSS using a preprocessor. # config.assets.css_compressor = :sass # Do not fallback to assets pipeline if a precompiled asset is missed. config.assets.compile = false - # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb - # Enable serving of images, stylesheets, and JavaScripts from an asset server. - # config.action_controller.asset_host = 'http://assets.example.com' + # config.asset_host = "http://assets.example.com" # Specifies the header that your server uses for sending files. - # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache - # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for Apache + # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX - # Mount Action Cable outside main process or domain + # Store uploaded files on the local file system (see config/storage.yml for options). + config.active_storage.service = :local + + # Mount Action Cable outside main process or domain. # config.action_cable.mount_path = nil - # config.action_cable.url = 'wss://example.com/cable' - # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + # config.action_cable.url = "wss://example.com/cable" + # config.action_cable.allowed_request_origins = [ "http://example.com", /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true - # Use the lowest log level to ensure availability of diagnostic information - # when problems arise. + # Include generic and useful information about system operation, but avoid logging too much + # information to avoid inadvertent exposure of personally identifiable information (PII). config.log_level = :info # Prepend all log lines with the following tags. - config.log_tags = [:request_id] + config.log_tags = [ :request_id ] # Use a different cache store in production. # config.cache_store = :mem_cache_store - # Use a real queuing backend for Active Job (and separate queues per environment) + # Use a real queuing backend for Active Job (and separate queues per environment). # config.active_job.queue_adapter = :resque - # config.active_job.queue_name_prefix = "qa_server_#{Rails.env}" - - # run jobs asynchronously in background (ok because only background jobs are related to monitoring status) - config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging + # config.active_job.queue_name_prefix = "qa_server_production" config.action_mailer.perform_caching = false @@ -69,15 +72,15 @@ # the I18n.default_locale when a translation cannot be found). config.i18n.fallbacks = true - # Send deprecation notices to registered listeners. - config.active_support.deprecation = :notify + # Don't log any deprecations. + config.active_support.report_deprecations = false # Use default logging formatter so that PID and timestamp are not suppressed. config.log_formatter = ::Logger::Formatter.new # Use a different logger for distributed setups. - # require 'syslog/logger' - # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + # require "syslog/logger" + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new "app-name") if ENV["RAILS_LOG_TO_STDOUT"].present? logger = ActiveSupport::Logger.new(STDOUT) @@ -87,8 +90,4 @@ # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false - - # Use lograge to tame rails logs - config.lograge.enabled = true - config.lograge.base_controller_class = ['ActionController::API', 'ActionController::Base'] end diff --git a/config/environments/test.rb b/config/environments/test.rb index 8e5cbde..6ea4d1e 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -1,32 +1,41 @@ +require "active_support/core_ext/integer/time" + +# The test environment is used exclusively to run your application's +# test suite. You never need to work with it otherwise. Remember that +# your test database is "scratch space" for the test suite and is wiped +# and recreated between test runs. Don't rely on the data there! + Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. - # The test environment is used exclusively to run your application's - # test suite. You never need to work with it otherwise. Remember that - # your test database is "scratch space" for the test suite and is wiped - # and recreated between test runs. Don't rely on the data there! + # Turn false under Spring and add config.action_view.cache_template_loading = true. config.cache_classes = true - # Do not eager load code on boot. This avoids loading your whole application - # just for the purpose of running a single test. If you are using a tool that - # preloads Rails for running tests, you may have to set it to true. - config.eager_load = false + # Eager loading loads your whole application. When running a single test locally, + # this probably isn't necessary. It's a good idea to do in a continuous integration + # system, or in some way before deploying your code. + config.eager_load = ENV["CI"].present? # Configure public file server for tests with Cache-Control for performance. config.public_file_server.enabled = true config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{1.hour.seconds.to_i}" + "Cache-Control" => "public, max-age=#{1.hour.to_i}" } # Show full error reports and disable caching. config.consider_all_requests_local = true config.action_controller.perform_caching = false + config.cache_store = :null_store # Raise exceptions instead of rendering exception templates. config.action_dispatch.show_exceptions = false # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false + + # Store uploaded files on the local file system in a temporary directory. + config.active_storage.service = :test + config.action_mailer.perform_caching = false # Tell Action Mailer not to deliver emails to the real world. @@ -37,6 +46,15 @@ # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr - # Raises error for missing translations - # config.action_view.raise_on_missing_translations = true + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true end diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb new file mode 100644 index 0000000..54f47cf --- /dev/null +++ b/config/initializers/content_security_policy.rb @@ -0,0 +1,25 @@ +# Be sure to restart your server when you modify this file. + +# Define an application-wide content security policy. +# See the Securing Rails Applications Guide for more information: +# https://guides.rubyonrails.org/security.html#content-security-policy-header + +# Rails.application.configure do +# config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end +# +# # Generate session nonces for permitted importmap and inline scripts +# config.content_security_policy_nonce_generator = ->(request) { request.session.id.to_s } +# config.content_security_policy_nonce_directives = %w(script-src) +# +# # Report violations without enforcing the policy. +# # config.content_security_policy_report_only = true +# end diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index 4a994e1..adc6568 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -1,4 +1,8 @@ # Be sure to restart your server when you modify this file. -# Configure sensitive parameters which will be filtered from the log file. -Rails.application.config.filter_parameters += [:password] +# Configure parameters to be filtered from the log file. Use this to limit dissemination of +# sensitive information. See the ActiveSupport::ParameterFilter documentation for supported +# notations and behaviors. +Rails.application.config.filter_parameters += [ + :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn +] diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index ac033bf..3860f65 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -4,13 +4,13 @@ # are locale specific, and you may define rules for as many different # locales as you wish. All of these examples are active by default: # ActiveSupport::Inflector.inflections(:en) do |inflect| -# inflect.plural /^(ox)$/i, '\1en' -# inflect.singular /^(ox)en/i, '\1' -# inflect.irregular 'person', 'people' +# inflect.plural /^(ox)$/i, "\\1en" +# inflect.singular /^(ox)en/i, "\\1" +# inflect.irregular "person", "people" # inflect.uncountable %w( fish sheep ) # end # These inflection rules are supported but not enabled by default: # ActiveSupport::Inflector.inflections(:en) do |inflect| -# inflect.acronym 'RESTful' +# inflect.acronym "RESTful" # end diff --git a/config/initializers/new_framework_defaults_7_0.rb b/config/initializers/new_framework_defaults_7_0.rb new file mode 100644 index 0000000..b13ef5e --- /dev/null +++ b/config/initializers/new_framework_defaults_7_0.rb @@ -0,0 +1,143 @@ +# Be sure to restart your server when you modify this file. +# +# This file eases your Rails 7.0 framework defaults upgrade. +# +# Uncomment each configuration one by one to switch to the new default. +# Once your application is ready to run with all new defaults, you can remove +# this file and set the `config.load_defaults` to `7.0`. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. +# https://guides.rubyonrails.org/upgrading_ruby_on_rails.html + +# `button_to` view helper will render `