From 5cf9cecd50166ae2c0f99112ca410e5ec3436ad8 Mon Sep 17 00:00:00 2001 From: Brianna Dardin Date: Mon, 4 Mar 2024 00:42:21 -0800 Subject: [PATCH] OD-1725 Load Authors Dynamically (#215) * OD-1725 Load Authors Dynamically * Added more tests for author/item errors and made some tweaks * Changed number pagination logic * Fixed column does not exist migration error * Removed all references to 'ariana' and replaced with 'test' as needed * Putting ariana.ao3.org back as archive host option --- app/controllers/authors_controller.rb | 24 ++-- app/controllers/stats_controller.rb | 52 +++------ app/javascript/actions/index.js | 2 +- .../components/pages/AuthorsPage.js | 33 +++++- .../pagination/AlphabeticalPagination.js | 16 +-- .../components/pagination/NumberPagination.js | 95 ++++++++++++++-- app/models/author.rb | 105 ++++++++++++++++++ app/models/concerns/item.rb | 100 +++++++++++++++++ app/views/archive_configs/edit.html.erb | 2 +- app/views/authors/index.html.erb | 4 +- config/initializers/constants.rb | 1 + config/routes.rb | 2 +- ...21222726_remove_author_id_from_chapters.rb | 4 +- db/schema.rb | 2 +- db/structure.sql | 2 +- lib/otw_archive/import_config.rb | 3 +- .../ansible/templates/archive_config.sql.j2 | 4 +- scripts/docker/init.sh | 2 +- spec/controllers/authors_controller_spec.rb | 8 +- spec/controllers/stats_controller_spec.rb | 27 ++--- spec/factories.rb | 6 +- spec/models/authors_spec.rb | 54 +++++---- spec/models/item_spec.rb | 57 ++++++++++ 23 files changed, 491 insertions(+), 114 deletions(-) create mode 100644 spec/models/item_spec.rb diff --git a/app/controllers/authors_controller.rb b/app/controllers/authors_controller.rb index 35dbd6d..37c9b39 100644 --- a/app/controllers/authors_controller.rb +++ b/app/controllers/authors_controller.rb @@ -2,6 +2,7 @@ class AuthorsController < ApplicationController require 'application_helper' + require "resolv" include OtwArchive include OtwArchive::Request @@ -11,13 +12,16 @@ class AuthorsController < ApplicationController # Prevent unhandled errors from returning the normal HTML page rescue_from StandardError, with: :render_standard_error_response + $page_size = 100 + def initialize super @client ||= otw_client end def index - @all_letters ||= Author.all_letters + @letter_counts ||= Author.letter_counts + @authors ||= current_authors end def author_letters @@ -39,7 +43,7 @@ def import_author current_user, processing_status: "importing") - response = author.import(@client, request.host_with_port) + response = author.import(@client, get_host(request)) message = "Processed import for #{author.name} with status #{response[:status]}: #{response[:messages].join(' ')}" processing_status = response[:author_imported] ? "imported" : "none" @@ -88,7 +92,7 @@ def check begin ApplicationHelper.broadcast_message("Checking #{author.name}", id, current_user, processing_status: "checking") - response = author.check(@client, request.host_with_port) + response = author.check(@client, get_host(request)) message = "Processed check for #{author.name} with status #{response[:status]}: #{response[:messages].join(' ')}" ApplicationHelper.broadcast_message(message, id, current_user, response: response) @@ -132,12 +136,18 @@ def otw_client OtwArchive::Client.new(import_config) end + def get_host(request) + if @client.config.archive_host.split("//")[1].split(":")[0] =~ Resolv::IPv4::Regex + host = @client.config.archive_host.dup.sub("3000", "3010") + else + host = request.host_with_port + end + host + end + def current_authors - max = 30 page = params[:page] || '1' letter = params[:letter] || 'A' - all_current_authors = Author.by_letter_with_items(letter) - @pages = (all_current_authors.size.to_f / max.to_f).ceil - all_current_authors[(page.to_i - 1) * 30, 30] + Author.get_letter(letter, page, $page_size) end end diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index a8ee913..7066ad9 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -9,27 +9,11 @@ def stats render json: stats.to_h.to_json, content_type: "application/json" end - def author_stats(authors) - authors_imported, authors_not_imported, authors_dni = Array.new(3) { [] } - authors.each do |a| - authors_imported << a if a.all_imported? - authors_dni << a if a.do_not_import - authors_not_imported << a if !a.all_imported? && !a.do_not_import - end - - OpenStruct.new( - all: authors.size, - imported: authors_imported.size, - not_imported: authors_not_imported.size, - dni: authors_dni.size - ) - end - - def grouped_author_stats(authors) + def grouped_author_stats(authors, imported) grouped_authors = authors.group_by { |author| author.name.downcase.first } all_imported = {} grouped_authors.each do |letter, group| - is_imported = group.all? { |a| a.all_imported? } + is_imported = group.all? { |a| imported.include?(a.id) } if letter =~ /[0-9a-zA-Z]/ all_imported[letter] = is_imported else @@ -49,32 +33,24 @@ def grouped_author_stats(authors) ) end - def story_stats(stories) - stories_imported, stories_dni, stories_not_imported = Array.new(3) { [] } - stories.each do |s| - stories_imported << s if s.imported - stories_dni << s if s.do_not_import - stories_not_imported << s if !s.imported && !s.do_not_import - end + def item_stats(type) OpenStruct.new( - all: stories.size, - imported: stories_imported.size, - not_imported: stories_not_imported.size, - dni: stories_dni.size + all: type.count, + imported: type.where(imported: true).count, + not_imported: type.where("imported = false AND do_not_import = false").count, + dni: type.where(do_not_import: true).count ) end - def gather_stats(authors = nil, stories = nil, story_links = nil) - authors ||= Author.with_stories_or_story_links - - stories ||= Story.all.to_a - story_links ||= StoryLink.all.to_a + def gather_stats + authors = Author.with_stories_or_story_links + imported = Author.all_imported @stats = OpenStruct.new( - authors: author_stats(authors), - letters: grouped_author_stats(authors), - stories: story_stats(stories), - story_links: story_stats(story_links) + authors: item_stats(Author), + letters: grouped_author_stats(authors, imported), + stories: item_stats(Story), + story_links: item_stats(StoryLink) ) end end diff --git a/app/javascript/actions/index.js b/app/javascript/actions/index.js index 4e9b438..01ad76e 100644 --- a/app/javascript/actions/index.js +++ b/app/javascript/actions/index.js @@ -13,7 +13,7 @@ export const CHECK_ITEM = "check_item"; export const DNI_ITEM = "dni_item"; -function getReq(endpoint) { +export function getReq(endpoint) { return axios.get(`/${sitekey}/${endpoint}`, { headers: { 'X-Requested-With': 'XMLHttpRequest' } }); } diff --git a/app/javascript/components/pages/AuthorsPage.js b/app/javascript/components/pages/AuthorsPage.js index f59f3c0..3021fa8 100644 --- a/app/javascript/components/pages/AuthorsPage.js +++ b/app/javascript/components/pages/AuthorsPage.js @@ -5,28 +5,52 @@ import Authors from "../items/Authors"; import NumberPagination from "../pagination/NumberPagination"; import Col from "react-bootstrap/Col"; import { authors_path } from "../../config"; +import { getReq } from "../../actions"; export default class AuthorsPage extends Component { constructor(props) { super(props); - this.state = this.props.data; + this.state = Object.assign({ pages: 0 }, this.props.data); } handleLetterChange = (letter) => { history.pushState(null, `Authors for ${letter}`, authors_path(letter)); - this.setState({ letter: letter, selectedAuthor: undefined }); + this.setState({ letter: letter, page: '1', selectedAuthor: undefined }); + this.getAuthors(letter, '1'); }; handlePageChange = (page) => { const letter = this.state.letter; history.pushState(null, `Authors for ${letter}, page ${page}`, authors_path(letter, page)); this.setState({ page: page, selectedAuthor: undefined }); + this.getAuthors(letter, page); }; handleAuthorSelect = (key) => { this.setState({ selectedAuthor: key }); }; + getAuthors = (letter, page) => { + const endpoint = `authors/letters/${letter}/${page}`; + getReq(endpoint).then(res => { + if (res.data) { + this.setState({ authors: res.data }); + this.setPages(); + } + }) + .catch(err => { + console.log(JSON.stringify(err)); + }) + } + + setPages() { + this.setState({ pages: Math.ceil(this.state.letter_counts[this.state.letter]["all"] / this.state.page_size) }); + } + + componentDidMount() { + this.setPages(); + } + componentDidUpdate() { if (this.state.selectedAuthor) { const element = document.getElementById(this.state.selectedAuthor); @@ -36,7 +60,8 @@ export default class AuthorsPage extends Component { render() { const alphabeticalPagination = ; const numberPagination = (this.state.pages > 1) ? @@ -51,7 +76,7 @@ export default class AuthorsPage extends Component { {alphabeticalPagination} {numberPagination} - + {numberPagination} {alphabeticalPagination} diff --git a/app/javascript/components/pagination/AlphabeticalPagination.js b/app/javascript/components/pagination/AlphabeticalPagination.js index 27b9382..ef94a46 100644 --- a/app/javascript/components/pagination/AlphabeticalPagination.js +++ b/app/javascript/components/pagination/AlphabeticalPagination.js @@ -22,10 +22,10 @@ export default class AlphabeticalPagination extends React.Component { }; render() { - const listItems = Object.entries(this.props.authors).map((kv) => { - const [ l, as ] = kv; - const numAuthors = as.length; - const authorsWithImports = as.filter(a => (a.s_to_import + a.l_to_import > 0) && !a.imported).length; + const listItems = Object.entries(this.props.letters).map((kv) => { + const [ l, counts ] = kv; + const numAuthors = counts.all; + const authorsWithImports = counts.imports; const isCurrent = (l === this.props.letter); const isDone = authorsWithImports === 0; @@ -47,7 +47,7 @@ export default class AlphabeticalPagination extends React.Component { } ); - const letters = Object.keys(this.props.authors); + const letters = Object.keys(this.props.letters); const letterIndex = letters.findIndex(x => x === this.props.letter); const prev = letterIndex - 1; const next = letterIndex + 1; @@ -63,10 +63,10 @@ export default class AlphabeticalPagination extends React.Component { Scroll to... - {Object.entries(this.props.authors).map((kv) => { - const [ l, authors ] = kv; + {Object.entries(this.props.letters).map((kv) => { + const [ l, counts ] = kv; if (l === this.props.letter) { - return authors.map(a => { + return this.props.authors.map(a => { const key = `author-${a.id}`; return this.handleAuthorSelect(e, key)}>{a.name} diff --git a/app/javascript/components/pagination/NumberPagination.js b/app/javascript/components/pagination/NumberPagination.js index 0aaef13..71ffbc4 100644 --- a/app/javascript/components/pagination/NumberPagination.js +++ b/app/javascript/components/pagination/NumberPagination.js @@ -1,7 +1,15 @@ import React, { Component } from "react"; import Pagination from "react-bootstrap/Pagination"; +import Dropdown from "react-bootstrap/Dropdown"; import { authors_path } from "../../config"; +const DOTS = "..."; + +const range = (start, end) => { + let length = end - start + 1; + return Array.from({ length }, (_, idx) => idx + start); +}; + export default class NumberPagination extends Component { handlePageChange = (e, p) => { @@ -9,16 +17,68 @@ export default class NumberPagination extends Component { this.props.onPageChange(p) }; + // Pagination logic adapted from https://www.freecodecamp.org/news/build-a-custom-pagination-component-in-react/ + getPaginationRange = (currentPage, totalPageCount, siblingCount, totalPageNumbers) => { + if (totalPageNumbers >= totalPageCount) { + return range(1, totalPageCount); + } + + const leftSiblingIndex = Math.max(currentPage - siblingCount, 1); + const rightSiblingIndex = Math.min( + currentPage + siblingCount, + totalPageCount + ); + + const shouldShowLeftDots = leftSiblingIndex > 2; + const shouldShowRightDots = rightSiblingIndex < totalPageCount - 2; + + const firstPageIndex = 1; + const lastPageIndex = totalPageCount; + + if (!shouldShowLeftDots && shouldShowRightDots) { + let leftItemCount = 3 + 2 * siblingCount; + let leftRange = range(1, leftItemCount); + + return [...leftRange, DOTS, totalPageCount]; + } + + if (shouldShowLeftDots && !shouldShowRightDots) { + let rightItemCount = 3 + 2 * siblingCount; + let rightRange = range( + totalPageCount - rightItemCount + 1, + totalPageCount + ); + return [firstPageIndex, DOTS, ...rightRange]; + } + + if (shouldShowLeftDots && shouldShowRightDots) { + let middleRange = range(leftSiblingIndex, rightSiblingIndex); + return [firstPageIndex, DOTS, ...middleRange, DOTS, lastPageIndex]; + } + } + render() { - const listItems = [...Array(this.props.pages).keys()].map(p => p + 1).map((p) => { - const isCurrent = (p === this.props.page); + const currentPageInt = parseInt(this.props.page); + const totalPagesInt = parseInt(this.props.pages); + const siblingCount = 3; + const totalPageNumbers = siblingCount + 5; + const paginationRange = this.getPaginationRange(currentPageInt, totalPagesInt, siblingCount, totalPageNumbers); + + let dotCount = 0; + const shownItems = paginationRange.map(p => { + if (p === DOTS) { + dotCount++; + return + } + + const isCurrent = (p === currentPageInt); const props = { active: isCurrent, onClick: e => this.handlePageChange(e, p) }; return {p} - } - ); + }); + const props = p => { const isValid = (p > 0 && p <= this.props.pages); return { @@ -28,16 +88,35 @@ export default class NumberPagination extends Component { } }; + const getDropdown = () => { + return ( + + Jump to page... + + {[...Array(totalPagesInt).keys()].map(p => p + 1).map((p) => { + if (p !== currentPageInt) { + return this.handlePageChange(e, p)}>{p} + } + })} + + + ) + } + const dropdown = (totalPageNumbers >= totalPagesInt) ? "" : getDropdown(); + return ( ) + + ) } } diff --git a/app/models/author.rb b/app/models/author.rb index 0a8b8c4..e97ec0d 100644 --- a/app/models/author.rb +++ b/app/models/author.rb @@ -35,6 +35,43 @@ def items_errors [story_errors, link_errors].compact.reduce([], :|) end + def self.all_errors(author_ids) + all_errors = Author.size_errors(author_ids) + item_errors = Item.all_errors(author_ids) + item_errors.map do |a_id, errors| + if all_errors.key?(a_id) + all_errors[a_id].concat errors + else + all_errors[a_id] = errors + end + end + all_errors + end + + def self.size_errors(author_ids) + author_errors = {} + author_names = Author.where("id in (#{author_ids})").pluck(:id, :name).to_h + + items = [ + {model: Story, label: :stories}, + {model: StoryLink, label: :bookmarks} + ] + + items.map do |item| + errors = item[:model].where("author_id in (#{author_ids})").group(:author_id).having("count(id) > #{NUMBER_OF_ITEMS}").count + errors.map do |a_id, count| + msg = "Author '#{author_names[a_id]}' has #{count} #{item[:label]} - the Archive can only import #{NUMBER_OF_ITEMS} at a time" + if author_errors.key?(a_id) + author_errors[a_id] << msg + else + author_errors[a_id] = [msg] + end + end + end + + author_errors + end + def coauthored_stories Story.where(coauthor_id: id) end @@ -71,6 +108,74 @@ def self.all_letters end.group_by { |a| a[:name][0].upcase } end + def self.all_imported + non_imported_stories = Item.auth_id_to_not_imported_dni(Story) + non_imported_links = Item.auth_id_to_not_imported_dni(StoryLink) + authors = [] + Author.all.map do |a| + s_to_import = non_imported_stories[a.id] || 0 + l_to_import = non_imported_links[a.id] || 0 + authors << a.id if s_to_import == 0 && l_to_import == 0 + end + authors + end + + def self.not_imported + non_imported_stories = Item.auth_id_to_not_imported_dni(Story) + non_imported_links = Item.auth_id_to_not_imported_dni(StoryLink) + authors = [] + Author.with_stories_or_story_links.map do |a| + s_to_import = non_imported_stories[a.id] || 0 + l_to_import = non_imported_links[a.id] || 0 + authors << a if s_to_import > 0 || l_to_import > 0 + end + authors + end + + def self.authors_by_letter(letter, page, page_size) + offset = (page.to_i - 1) * page_size + Author.left_outer_joins(:stories, :story_links).where( + "substr(upper(name),1,1) = '#{letter}' and (stories.id IS NOT NULL or story_links.id IS NOT NULL)" + ).order(:name).limit(page_size).offset(offset).distinct + end + + def self.get_letter(letter, page, page_size) + authors = Author.authors_by_letter(letter, page, page_size).to_a + author_ids = authors.map(&:id).join(",") + + id_to_stories = Item.auth_id_to_not_imported(Story, author_ids) + id_to_links = Item.auth_id_to_not_imported(StoryLink, author_ids) + errors = Author.all_errors(author_ids) + + authors.map do |a| + { + id: a.id, + name: a.name, + imported: a.imported, + s_to_import: id_to_stories[a.id] || 0, + l_to_import: id_to_links[a.id] || 0, + errors: errors[a.id] || [] + } + end + end + + def self.letter_counts + letters = Author.with_stories_or_story_links.group_by { |a| a[:name][0].upcase } + not_imported_letters = Author.not_imported.group_by { |a| a[:name][0].upcase } + + all_authors = Hash[letters.map{|k,v| [k,v.size]}] + not_imported = Hash[not_imported_letters.map{|k,v| [k, v.size]}] + + letter_hash = {} + all_authors.map do |letter,count| + letter_hash[letter] = { + all: count, + imports: not_imported[letter] || 0 + } + end + letter_hash + end + def import(client, host) if do_not_import response = diff --git a/app/models/concerns/item.rb b/app/models/concerns/item.rb index 9e3ce8b..f6b3195 100644 --- a/app/models/concerns/item.rb +++ b/app/models/concerns/item.rb @@ -33,6 +33,106 @@ def item_errors errors end + def self.all_errors(author_ids) + author_errors = {} + + items = [ + {model: Story, label: :story}, + {model: StoryLink, label: :bookmark} + ] + + item_cols = [ + {col: :summary, max: SUMMARY_LENGTH}, + {col: :notes, max: NOTES_LENGTH} + ] + + items.map do |item| + author_errors = Item.iterate_errors(Item.missing_fandom(item[:model], item[:label], author_ids), author_errors) + item_cols.map do |field| + author_errors = Item.iterate_errors(Item.too_long_errors(item[:model], item[:label], field[:col], field[:max], author_ids), author_errors) + end + end + + chapter_params = [ + {col: :notes, max: NOTES_LENGTH}, + {col: :text, max: CHAPTER_LENGTH} + ] + + chapter_params.map do |field| + author_errors = Item.iterate_errors(Item.chapter_errors(field[:col], field[:max], author_ids), author_errors) + end + + author_errors + end + + def self.iterate_errors(item_errors, author_errors) + item_errors.map do |a_id, errors| + if author_errors.key?(a_id) + author_errors[a_id].concat errors + else + author_errors[a_id] = errors + end + end + author_errors + end + + def self.missing_fandom(type_model, type_sym, author_ids) + where = Item.get_auth_id_query("(fandoms is null OR length(fandoms) = 0)", author_ids) + fandom_errors = type_model.where(where).group_by { |item| item[:author_id] } + missing_fandom_text = Proc.new do |type_sym, col, item| + "Fandom for #{type_sym} '#{item.title}' is missing" + end + Item.parse_author_errors(fandom_errors, type_sym, :fandoms, missing_fandom_text) + end + + def self.too_long_errors(type_model, type_sym, col, max, author_ids) + where = Item.get_auth_id_query("#{col} is not null AND length(#{col}) > #{max}", author_ids) + length_errors = type_model.where(where).group_by { |item| item[:author_id] } + too_long_text = Proc.new do |type_sym, col, item| + "#{col.capitalize} for #{type_sym} '#{item.title}' is too long (#{item[col].length})" + end + Item.parse_author_errors(length_errors, type_sym, col, too_long_text) + end + + def self.chapter_errors(col, max, author_ids) + where = Item.get_auth_id_query("chapters.#{col} is not null AND length(chapters.#{col}) > #{max}", author_ids).dup.sub("author_id", "stories.author_id") + length_errors = Chapter.joins(:story).where(where).select(Arel.sql("chapters.*, stories.author_id as a_id, stories.title as s_title")).group_by { |c| c[:a_id] } + chapter_text = Proc.new do |type_sym, col, item| + "#{col.capitalize} for #{type_sym} #{item.position} in story '#{item.s_title}' is too long (#{item[col].length})" + end + Item.parse_author_errors(length_errors, :chapter, col, chapter_text) + end + + def self.parse_author_errors(item_errors, type_sym, col, text_proc) + author_errors = {} + item_errors.map do |a_id, errors| + if errors.size > 0 + author_errors[a_id] = [] + errors.map do |item| + author_errors[a_id] << text_proc.call(type_sym, col, item) + end + end + end + author_errors + end + + def self.auth_id_to_not_imported(type, author_ids = nil) + where = Item.get_auth_id_query("imported = false", author_ids) + type.where(where).group(:author_id).count + end + + def self.auth_id_to_not_imported_dni(type, author_ids = nil) + where = Item.get_auth_id_query("imported = false AND do_not_import = false", author_ids) + type.where(where).group(:author_id).count + end + + def self.get_auth_id_query(where, author_ids = nil) + if !author_ids.nil? && author_ids.split(",").length > 0 + where += " AND author_id IN (#{author_ids})" + end + where + end + def self.items_responses(ao3_response, check = false) response = {} has_success = ao3_response[0][:success] diff --git a/app/views/archive_configs/edit.html.erb b/app/views/archive_configs/edit.html.erb index 37a3b10..f509e25 100644 --- a/app/views/archive_configs/edit.html.erb +++ b/app/views/archive_configs/edit.html.erb @@ -16,7 +16,7 @@ <% end %>
- <% host_list = ENV['RAILS_ENV'] == "production" ? [:ariana, :test, :live] : [:local, :ariana, :test, :live] %> + <% host_list = ENV['RAILS_ENV'] == "production" ? [:test, :live] : [:local, :test, :live] %> <%= f.label :host %> <%= f.select :host, host_list, {}, class: "form-control" %> Warning: changing this will clear imported and do not import flags and AO3 diff --git a/app/views/authors/index.html.erb b/app/views/authors/index.html.erb index 28680e1..638203b 100644 --- a/app/views/authors/index.html.erb +++ b/app/views/authors/index.html.erb @@ -1,6 +1,8 @@ <%= react_component("App", { - all_letters: @all_letters, + letter_counts: @letter_counts, + authors: @authors, letter: params[:letter] || 'A', + page_size: $page_size, page: params[:page] || '1', logo_path: asset_path("Opendoors.png"), config: @api_config.merge(@archive_config), diff --git a/config/initializers/constants.rb b/config/initializers/constants.rb index 522caf2..1d08241 100644 --- a/config/initializers/constants.rb +++ b/config/initializers/constants.rb @@ -1,4 +1,5 @@ SUMMARY_LENGTH = 1250 +NOTES_LENGTH = 5000 CHAPTER_LENGTH = 510_000 NUMBER_OF_ITEMS = 200 BROADCAST_CHANNEL = APP_CONFIG[:sitekey] \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 23c6d5d..27fa90b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -19,7 +19,7 @@ # AJAX end points get "authors/letters" => "authors#author_letters" - get "authors/letters/:letter" => "authors#authors" + get "authors/letters/:letter/:page" => "authors#authors" post "authors/import/:author_id" => "authors#import_author" get "authors/check/:author_id" => "authors#check" diff --git a/db/migrate/20231021222726_remove_author_id_from_chapters.rb b/db/migrate/20231021222726_remove_author_id_from_chapters.rb index f7a20e3..ece4dbf 100644 --- a/db/migrate/20231021222726_remove_author_id_from_chapters.rb +++ b/db/migrate/20231021222726_remove_author_id_from_chapters.rb @@ -1,5 +1,7 @@ class RemoveAuthorIdFromChapters < ActiveRecord::Migration[5.2] def change - remove_column :chapters, :authorID, :integer + if ActiveRecord::Base.connection.column_exists?(:chapters, :authorID) + remove_column :chapters, :authorID + end end end diff --git a/db/schema.rb b/db/schema.rb index 5c4437a..3624555 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -22,7 +22,7 @@ t.boolean "post_preview", default: false, null: false t.string "archivist", limit: 100, default: "testy", null: false t.string "collection_name" - t.string "host", limit: 15, default: "ariana" + t.string "host", limit: 15, default: "test" t.index ["id"], name: "id_UNIQUE", unique: true t.index ["key"], name: "Key_UNIQUE", unique: true end diff --git a/db/structure.sql b/db/structure.sql index e0d45bc..cce27f1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -36,7 +36,7 @@ CREATE TABLE `archive_configs` ( `post_preview` tinyint(1) NOT NULL DEFAULT '0', `archivist` varchar(100) NOT NULL DEFAULT 'testy', `collection_name` varchar(255) DEFAULT NULL, - `host` varchar(15) DEFAULT 'ariana', + `host` varchar(15) DEFAULT 'test', PRIMARY KEY (`id`), UNIQUE KEY `id_UNIQUE` (`id`), UNIQUE KEY `Key_UNIQUE` (`key`) diff --git a/lib/otw_archive/import_config.rb b/lib/otw_archive/import_config.rb index 4cedb94..3fb46c1 100644 --- a/lib/otw_archive/import_config.rb +++ b/lib/otw_archive/import_config.rb @@ -1,5 +1,6 @@ module OtwArchive class ImportConfig + require "resolv" attr_reader :archive_host, :token, :restricted, :override_tags, :detect_tags, :encoding, :archive_config @@ -17,7 +18,7 @@ def initialize(archive_host, token, restricted = true, override_tags = true, det end def isHttp?(archive_host) - archive_host.include?("ariana.archiveofourown.org") || archive_host.include?("localhost") + archive_host.include?("ariana.archiveofourown.org") || archive_host.include?("localhost") || archive_host.split(":")[0] =~ Resolv::IPv4::Regex end end end # OtwArchive diff --git a/scripts/ansible/templates/archive_config.sql.j2 b/scripts/ansible/templates/archive_config.sql.j2 index 00aa0ef..6f78ba7 100644 --- a/scripts/ansible/templates/archive_config.sql.j2 +++ b/scripts/ansible/templates/archive_config.sql.j2 @@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS `archive_configs` ( `post_preview` tinyint(1) NOT NULL DEFAULT '0', `archivist` varchar(100) NOT NULL DEFAULT 'testy', `collection_name` varchar(255) DEFAULT NULL, - `host` varchar(15) DEFAULT 'ariana', + `host` varchar(15) DEFAULT 'test', PRIMARY KEY (`id`), UNIQUE KEY `id_UNIQUE` (`id`), UNIQUE KEY `Key_UNIQUE` (`key`) @@ -23,7 +23,7 @@ INSERT IGNORE INTO `archive_configs` `archivist`, `collection_name`, `host`) VALUES (1, "{{ sitekey }}", "{{ name }}", "Testing", "OD STORY NOTE", "OD BOOKMARK NOTE", 0, 0, - "testy", "opendoorstestcollection", "ariana"); + "testy", "opendoorstestcollection", "test"); -- -- Table structure for table `audits` diff --git a/scripts/docker/init.sh b/scripts/docker/init.sh index 45f0eee..8c5f306 100755 --- a/scripts/docker/init.sh +++ b/scripts/docker/init.sh @@ -49,7 +49,7 @@ cp scripts/ansible/templates/archive_config.sql.j2 $SQL_FILE sed -i'' -e 's/{{ sitekey }}/opendoorstempsite/g' $SQL_FILE sed -i'' -e 's/{{ name }}/Open Doors Temp Site/g' $SQL_FILE -sed -i'' -e 's/ariana/local/g' $SQL_FILE +sed -i'' -e 's/test/local/g' $SQL_FILE #Auto-load sample SQL file docker-compose exec -T db mysql -h db -uroot -p$MYSQL_PASS opendoorstempsite < $SQL_FILE \ No newline at end of file diff --git a/spec/controllers/authors_controller_spec.rb b/spec/controllers/authors_controller_spec.rb index 1bbc17a..9dfbf5d 100644 --- a/spec/controllers/authors_controller_spec.rb +++ b/spec/controllers/authors_controller_spec.rb @@ -25,12 +25,14 @@ it "renders the authors template with a list of letters and authors" do get :index - letters = assigns(:all_letters) + letters = assigns(:letter_counts) expect(letters).to be_a Hash expect(letters.keys).to eq ["A"] - expect(letters["A"][0][:name]).to eq author1.name + authors = assigns(:authors) + expect(authors).to be_a Array + expect(authors[0][:name]).to eq author1.name end - + it "lists authors with stories and bookmarks" do get :author_letters expect(response.status).to be 200 diff --git a/spec/controllers/stats_controller_spec.rb b/spec/controllers/stats_controller_spec.rb index c50dc00..44dcd78 100644 --- a/spec/controllers/stats_controller_spec.rb +++ b/spec/controllers/stats_controller_spec.rb @@ -21,9 +21,8 @@ context "stats methods" do it "returns the author stats" do - author = Author.new(name: "Name", email: "foo@ao3.org") - authors = [author] - result = controller.author_stats(authors) + author = create(:author, imported: true) + result = controller.item_stats(Author) expect(result).to be_a(OpenStruct) expect(result.all).to eq 1 expect(result.imported).to eq 1 @@ -32,10 +31,9 @@ end it "returns the story stats" do - author = Author.new(name: "Name", email: "foo@ao3.org") - story = Story.new(title: "Name", author: author) - stories = [story] - result = controller.story_stats(stories) + author = create(:author) + story = create(:story, author_id: author.id, audit_comment: "Test") + result = controller.item_stats(Story) expect(result).to be_a(OpenStruct) expect(result.all).to eq 1 expect(result.imported).to eq 0 @@ -44,10 +42,9 @@ end it "returns the story link stats" do - author = Author.new(name: "Name", email: "foo@ao3.org") - story_link = StoryLink.new(title: "Name", author: author) - story_links = [story_link] - result = controller.story_stats(story_links) + author = create(:author) + story_link = create(:story_link, author_id: author.id, audit_comment: "Test") + result = controller.item_stats(StoryLink) expect(result).to be_a(OpenStruct) expect(result.all).to eq 1 expect(result.imported).to eq 0 @@ -56,10 +53,10 @@ end it "returns all the stats" do - author = Author.new(name: "Name", email: "foo@ao3.org") - story = Story.new(title: "Name", author: author) - story_link = StoryLink.new(title: "Name", author: author) - result = controller.gather_stats([author], [story], [story_link]) + author = create(:author) + story = create(:story, author_id: author.id, audit_comment: "Test") + story_link = create(:story_link, author_id: author.id, audit_comment: "Test") + result = controller.gather_stats expect(result).to be_a(OpenStruct) expect(result.authors.all).to eq 1 expect(result.letters.all).to eq 1 diff --git a/spec/factories.rb b/spec/factories.rb index 43dd793..a390f9a 100644 --- a/spec/factories.rb +++ b/spec/factories.rb @@ -36,7 +36,11 @@ end end - factory :chapter + factory :chapter do + story + id { generate :id } + audit_comment {"Test"} + end factory :archive_config do key { APP_CONFIG[:sitekey] } diff --git a/spec/models/authors_spec.rb b/spec/models/authors_spec.rb index 776753a..c2b355b 100644 --- a/spec/models/authors_spec.rb +++ b/spec/models/authors_spec.rb @@ -53,36 +53,52 @@ end describe "all_imported?" do - let(:story) {Story.new(imported: false, do_not_import: false)} - let(:story_link) { StoryLink.new(imported: false, do_not_import: false) } - let(:imported_story) { Story.new(imported: true, do_not_import: false) } - let(:imported_story_link) { StoryLink.new(imported: true, do_not_import: false) } - let(:do_not_import_story) { Story.new(imported: false, do_not_import: true) } - let(:do_not_import_story_link) { StoryLink.new(imported: false, do_not_import: true) } - it "returns true if both stories and story links are empty" do - imported_author = Author.new(stories: [], story_links: []) - expect(imported_author.all_imported?).to eq true + author = create(:author) + expect(Author.all_imported.include?(author.id)).to eq true end it "returns true if all stories are imported or marked as do not import and story links are empty" do - imported_author = Author.new(stories: [imported_story, do_not_import_story], story_links: []) - expect(imported_author.all_imported?).to eq true + author = create(:author) + imported_story = create(:story, author_id: author.id, imported: true, audit_comment: "Test") + do_not_import_story = create(:story, author_id: author.id, do_not_import: true, audit_comment: "Test") + expect(Author.all_imported.include?(author.id)).to eq true end it "returns true if stories are empty and all story links are imported or marked as do not import" do - imported_author = Author.new(stories: [], story_links: [imported_story_link, do_not_import_story_link]) - expect(imported_author.all_imported?).to eq true + author = create(:author) + imported_story_link = create(:story_link, author_id: author.id, imported: true, audit_comment: "Test") + do_not_import_story_link = create(:story_link, author_id: author.id, do_not_import: true, audit_comment: "Test") + expect(Author.all_imported.include?(author.id)).to eq true end it "returns false if only some stories are imported or marked as do not import and story links are empty" do - imported_author = Author.new(stories: [story, imported_story], story_links: []) - expect(imported_author.all_imported?).to eq false + author = create(:author) + story = create(:story, author_id: author.id, audit_comment: "Test") + imported_story = create(:story, author_id: author.id, imported: true, audit_comment: "Test") + expect(Author.all_imported.include?(author.id)).to eq false end it "returns false if stories are empty and only some story links are imported or marked as do not import" do - imported_author = Author.new(stories: [], story_links: [story_link, do_not_import_story_link]) - expect(imported_author.all_imported?).to eq false + author = create(:author) + story_link = create(:story_link, author_id: author.id, audit_comment: "Test") + imported_story_link = create(:story_link, author_id: author.id, imported: true, audit_comment: "Test") + expect(Author.all_imported.include?(author.id)).to eq false end it "returns false if no stories and no story links are imported or marked as do not import" do - imported_author = Author.new(stories: [story], story_links: [story_link]) - expect(imported_author.all_imported?).to eq false + author = create(:author) + story = create(:story, author_id: author.id, audit_comment: "Test") + story_link = create(:story_link, author_id: author.id, audit_comment: "Test") + expect(Author.all_imported.include?(author.id)).to eq false end end + + it 'returns too many item errors' do + stub_const("NUMBER_OF_ITEMS", 4) + author = create(:author, name: "testy") + 5.times { + story = create(:story, author_id: author.id, audit_comment: "Test") + story_link = create(:story_link, author_id: author.id, audit_comment: "Test") + } + errors = Author.all_errors(author.id.to_s) + expect(errors.key?(author.id)).to eq true + expect(errors[author.id].include?("Author 'testy' has 5 stories - the Archive can only import 4 at a time")).to eq true + expect(errors[author.id].include?("Author 'testy' has 5 bookmarks - the Archive can only import 4 at a time")).to eq true + end end diff --git a/spec/models/item_spec.rb b/spec/models/item_spec.rb new file mode 100644 index 0000000..444d5b9 --- /dev/null +++ b/spec/models/item_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Item, type: :model do + it 'returns summary too long errors' do + stub_const("SUMMARY_LENGTH", 4) + author = create(:author) + story = create(:story, author_id: author.id, title: "title", summary: "Longer than 4 characters", audit_comment: "Test") + story_link = create(:story_link, author_id: author.id, title: "title", summary: "Longer than 4 characters", audit_comment: "Test") + errors = Item.all_errors(author.id.to_s) + expect(errors.key?(author.id)).to eq true + expect(errors[author.id].include?("Summary for story 'title' is too long (24)")).to eq true + expect(errors[author.id].include?("Summary for bookmark 'title' is too long (24)")).to eq true + end + + it 'returns notes too long errors' do + stub_const("NOTES_LENGTH", 4) + author = create(:author) + story = create(:story, author_id: author.id, title: "title", notes: "Longer than 4 characters", audit_comment: "Test") + story_link = create(:story_link, author_id: author.id, title: "title", notes: "Longer than 4 characters", audit_comment: "Test") + errors = Item.all_errors(author.id.to_s) + expect(errors.key?(author.id)).to eq true + expect(errors[author.id].include?("Notes for story 'title' is too long (24)")).to eq true + expect(errors[author.id].include?("Notes for bookmark 'title' is too long (24)")).to eq true + end + + it 'returns missing fandoms errors' do + author = create(:author) + story = create(:story, author_id: author.id, title: "title", audit_comment: "Test") + story_link = create(:story_link, author_id: author.id, title: "title", audit_comment: "Test") + errors = Item.all_errors(author.id.to_s) + expect(errors.key?(author.id)).to eq true + expect(errors[author.id].include?("Fandom for story 'title' is missing")).to eq true + expect(errors[author.id].include?("Fandom for bookmark 'title' is missing")).to eq true + end + + it 'returns chapter notes too long errors' do + stub_const("NOTES_LENGTH", 4) + author = create(:author) + story = create(:story, author_id: author.id, title: "title", audit_comment: "Test") + chapter = create(:chapter, story_id: story.id, position: 1, notes: "Longer than 4 characters", audit_comment: "Test") + errors = Item.all_errors(author.id.to_s) + expect(errors.key?(author.id)).to eq true + expect(errors[author.id].include?("Notes for chapter 1 in story 'title' is too long (24)")).to eq true + end + + it 'returns chapter text too long errors' do + stub_const("CHAPTER_LENGTH", 4) + author = create(:author) + story = create(:story, author_id: author.id, title: "title", audit_comment: "Test") + chapter = create(:chapter, story_id: story.id, position: 1, text: "Longer than 4 characters", audit_comment: "Test") + errors = Item.all_errors(author.id.to_s) + expect(errors.key?(author.id)).to eq true + expect(errors[author.id].include?("Text for chapter 1 in story 'title' is too long (24)")).to eq true + end +end