OD-1725 Load Authors Dynamically (#215)

* OD-1725 Load Authors Dynamically

* Added more tests for author/item errors and made some tweaks

* Changed number pagination logic

* Fixed column does not exist migration error

* Removed all references to 'ariana' and replaced with 'test' as needed

* Putting ariana.ao3.org back as archive host option

app/controllers/authors_controller.rb

@@ -2,6 +2,7 @@
 
 class AuthorsController < ApplicationController
   require 'application_helper'
+  require "resolv"
 
   include OtwArchive
   include OtwArchive::Request
@@ -11,13 +12,16 @@ class AuthorsController < ApplicationController
   # Prevent unhandled errors from returning the normal HTML page
   rescue_from StandardError, with: :render_standard_error_response
 
+  $page_size = 100
+
   def initialize
     super
     @client ||= otw_client
   end
 
   def index
-    @all_letters ||= Author.all_letters
+    @letter_counts ||= Author.letter_counts
+    @authors ||= current_authors
   end
 
   def author_letters
@@ -39,7 +43,7 @@ def import_author
         current_user,
         processing_status: "importing")
 
-      response = author.import(@client, request.host_with_port)
+      response = author.import(@client, get_host(request))
 
       message = "Processed import for #{author.name} with status #{response[:status]}: #{response[:messages].join(' ')}"
       processing_status = response[:author_imported] ? "imported" : "none"
@@ -88,7 +92,7 @@ def check
     begin
       ApplicationHelper.broadcast_message("Checking #{author.name}", id, current_user, processing_status: "checking")
 
-      response = author.check(@client, request.host_with_port)
+      response = author.check(@client, get_host(request))
 
       message = "Processed check for #{author.name} with status #{response[:status]}: #{response[:messages].join(' ')}"
       ApplicationHelper.broadcast_message(message, id, current_user, response: response)
@@ -132,12 +136,18 @@ def otw_client
     OtwArchive::Client.new(import_config)
   end
 
+  def get_host(request)
+    if @client.config.archive_host.split("//")[1].split(":")[0] =~ Resolv::IPv4::Regex
+      host = @client.config.archive_host.dup.sub("3000", "3010")
+    else
+      host = request.host_with_port
+    end
+    host
+  end
+
   def current_authors
-    max = 30
     page = params[:page] || '1'
     letter = params[:letter] || 'A'
-    all_current_authors = Author.by_letter_with_items(letter)
-    @pages = (all_current_authors.size.to_f / max.to_f).ceil
-    all_current_authors[(page.to_i - 1) * 30, 30]
+    Author.get_letter(letter, page, $page_size)
   end
 end

app/controllers/stats_controller.rb

@@ -9,27 +9,11 @@ def stats
     render json: stats.to_h.to_json, content_type: "application/json"
   end
 
-  def author_stats(authors)
-    authors_imported, authors_not_imported, authors_dni = Array.new(3) { [] }
-    authors.each do |a|
-      authors_imported << a if a.all_imported?
-      authors_dni << a if a.do_not_import
-      authors_not_imported << a if !a.all_imported? && !a.do_not_import
-    end
-
-    OpenStruct.new(
-        all: authors.size,
-        imported: authors_imported.size,
-        not_imported: authors_not_imported.size,
-        dni: authors_dni.size
-    )
-  end
-
-  def grouped_author_stats(authors)
+  def grouped_author_stats(authors, imported)
     grouped_authors = authors.group_by { |author| author.name.downcase.first }
     all_imported = {}
     grouped_authors.each do |letter, group|
-      is_imported = group.all? { |a| a.all_imported? }
+      is_imported = group.all? { |a| imported.include?(a.id) }
       if letter =~ /[0-9a-zA-Z]/
         all_imported[letter] = is_imported
       else
@@ -49,32 +33,24 @@ def grouped_author_stats(authors)
     )
   end
 
-  def story_stats(stories)
-    stories_imported, stories_dni, stories_not_imported = Array.new(3) { [] }
-    stories.each do |s|
-      stories_imported << s if s.imported
-      stories_dni << s if s.do_not_import
-      stories_not_imported << s if !s.imported && !s.do_not_import
-    end
+  def item_stats(type)
     OpenStruct.new(
-        all: stories.size,
-        imported: stories_imported.size,
-        not_imported: stories_not_imported.size,
-        dni: stories_dni.size
+        all: type.count,
+        imported: type.where(imported: true).count,
+        not_imported: type.where("imported = false AND do_not_import = false").count,
+        dni: type.where(do_not_import: true).count
     )
   end
 
-  def gather_stats(authors = nil, stories = nil, story_links = nil)
-    authors ||= Author.with_stories_or_story_links
-
-    stories ||= Story.all.to_a
-    story_links ||= StoryLink.all.to_a
+  def gather_stats
+    authors = Author.with_stories_or_story_links
+    imported = Author.all_imported
 
     @stats = OpenStruct.new(
-      authors: author_stats(authors),
-      letters: grouped_author_stats(authors),
-      stories: story_stats(stories),
-      story_links: story_stats(story_links)
+      authors: item_stats(Author),
+      letters: grouped_author_stats(authors, imported),
+      stories: item_stats(Story),
+      story_links: item_stats(StoryLink)
     )
   end
 end

app/javascript/actions/index.js

@@ -13,7 +13,7 @@ export const CHECK_ITEM = "check_item";
 export const DNI_ITEM = "dni_item";
 
 
-function getReq(endpoint) {
+export function getReq(endpoint) {
   return axios.get(`/${sitekey}/${endpoint}`,
     { headers: { 'X-Requested-With': 'XMLHttpRequest' } });
 }

app/javascript/components/pages/AuthorsPage.js

@@ -5,28 +5,52 @@ import Authors from "../items/Authors";
 import NumberPagination from "../pagination/NumberPagination";
 import Col from "react-bootstrap/Col";
 import { authors_path } from "../../config";
+import { getReq } from "../../actions";
 
 export default class AuthorsPage extends Component {
   constructor(props) {
     super(props);
-    this.state = this.props.data;
+    this.state = Object.assign({ pages: 0 }, this.props.data);
   }
 
   handleLetterChange = (letter) => {
     history.pushState(null, `Authors for ${letter}`, authors_path(letter));
-    this.setState({ letter: letter, selectedAuthor: undefined });
+    this.setState({ letter: letter, page: '1', selectedAuthor: undefined });
+    this.getAuthors(letter, '1');
   };
 
   handlePageChange = (page) => {
     const letter = this.state.letter;
     history.pushState(null, `Authors for ${letter}, page ${page}`, authors_path(letter, page));
     this.setState({ page: page, selectedAuthor: undefined });
+    this.getAuthors(letter, page);
   };
 
   handleAuthorSelect = (key) => {
     this.setState({ selectedAuthor: key });
   };
 
+  getAuthors = (letter, page) => {
+    const endpoint = `authors/letters/${letter}/${page}`;
+    getReq(endpoint).then(res => {
+      if (res.data) {
+        this.setState({ authors: res.data });
+        this.setPages();
+      }
+    })
+      .catch(err => {
+        console.log(JSON.stringify(err));
+      })
+  }
+
+  setPages() {
+    this.setState({ pages: Math.ceil(this.state.letter_counts[this.state.letter]["all"] / this.state.page_size) });
+  }
+
+  componentDidMount() {
+    this.setPages();
+  }
+
   componentDidUpdate() {
     if (this.state.selectedAuthor) {
       const element = document.getElementById(this.state.selectedAuthor);
@@ -36,7 +60,8 @@ export default class AuthorsPage extends Component {
 
   render() {
     const alphabeticalPagination = <AlphabeticalPagination letter={this.state.letter}
-                                                           authors={this.state.all_letters}
+                                                           letters={this.state.letter_counts}
+                                                           authors={this.state.authors}
                                                            onAuthorSelect={this.handleAuthorSelect}
                                                            onLetterChange={this.handleLetterChange}/>;
     const numberPagination = (this.state.pages > 1) ?
@@ -51,7 +76,7 @@ export default class AuthorsPage extends Component {
         {alphabeticalPagination}
         {numberPagination}
 
-        <Authors letter={this.state.letter} user={this.props.user} authors={this.state.all_letters[this.state.letter]}/>
+        <Authors letter={this.state.letter} user={this.props.user} authors={this.state.authors}/>
 
         {numberPagination}
         {alphabeticalPagination}

app/javascript/components/pagination/AlphabeticalPagination.js

@@ -22,10 +22,10 @@ export default class AlphabeticalPagination extends React.Component {
   };
 
   render() {
-    const listItems = Object.entries(this.props.authors).map((kv) => {
-        const [ l, as ] = kv;
-        const numAuthors = as.length;
-        const authorsWithImports = as.filter(a => (a.s_to_import + a.l_to_import > 0) && !a.imported).length;
+    const listItems = Object.entries(this.props.letters).map((kv) => {
+        const [ l, counts ] = kv;
+        const numAuthors = counts.all;
+        const authorsWithImports = counts.imports;
         const isCurrent = (l === this.props.letter);
         const isDone = authorsWithImports === 0;
 
@@ -47,7 +47,7 @@ export default class AlphabeticalPagination extends React.Component {
       }
     );
 
-    const letters = Object.keys(this.props.authors);
+    const letters = Object.keys(this.props.letters);
     const letterIndex = letters.findIndex(x => x === this.props.letter);
     const prev = letterIndex - 1;
     const next = letterIndex + 1;
@@ -63,10 +63,10 @@ export default class AlphabeticalPagination extends React.Component {
           <Dropdown className="page-item">
             <Dropdown.Toggle className="page-link" id="dropdown-basic-button">Scroll to...</Dropdown.Toggle>
             <Dropdown.Menu>
-              {Object.entries(this.props.authors).map((kv) => {
-                const [ l, authors ] = kv;
+              {Object.entries(this.props.letters).map((kv) => {
+                const [ l, counts ] = kv;
                 if (l === this.props.letter) {
-                  return authors.map(a => {
+                  return this.props.authors.map(a => {
                     const key = `author-${a.id}`;
                     return <Dropdown.Item key={`${key}-link`}
                                           onClick={e => this.handleAuthorSelect(e, key)}>{a.name}</Dropdown.Item>

app/javascript/components/pagination/NumberPagination.js

@@ -1,24 +1,84 @@
 import React, { Component } from "react";
 import Pagination from "react-bootstrap/Pagination";
+import Dropdown from "react-bootstrap/Dropdown";
 import { authors_path } from "../../config";
 
+const DOTS = "...";
+
+const range = (start, end) => {
+  let length = end - start + 1;
+  return Array.from({ length }, (_, idx) => idx + start);
+};
+
 export default class NumberPagination extends Component {
 
   handlePageChange = (e, p) => {
     e.preventDefault();
     this.props.onPageChange(p)
   };
 
+  // Pagination logic adapted from https://www.freecodecamp.org/news/build-a-custom-pagination-component-in-react/
+  getPaginationRange = (currentPage, totalPageCount, siblingCount, totalPageNumbers) => {
+    if (totalPageNumbers >= totalPageCount) {
+      return range(1, totalPageCount);
+    }
+
+    const leftSiblingIndex = Math.max(currentPage - siblingCount, 1);
+    const rightSiblingIndex = Math.min(
+      currentPage + siblingCount,
+      totalPageCount
+    );
+
+    const shouldShowLeftDots = leftSiblingIndex > 2;
+    const shouldShowRightDots = rightSiblingIndex < totalPageCount - 2;
+
+    const firstPageIndex = 1;
+    const lastPageIndex = totalPageCount;
+
+    if (!shouldShowLeftDots && shouldShowRightDots) {
+      let leftItemCount = 3 + 2 * siblingCount;
+      let leftRange = range(1, leftItemCount);
+
+      return [...leftRange, DOTS, totalPageCount];
+    }
+
+    if (shouldShowLeftDots && !shouldShowRightDots) {
+      let rightItemCount = 3 + 2 * siblingCount;
+      let rightRange = range(
+        totalPageCount - rightItemCount + 1,
+        totalPageCount
+      );
+      return [firstPageIndex, DOTS, ...rightRange];
+    }
+
+    if (shouldShowLeftDots && shouldShowRightDots) {
+      let middleRange = range(leftSiblingIndex, rightSiblingIndex);
+      return [firstPageIndex, DOTS, ...middleRange, DOTS, lastPageIndex];
+    }
+  }
+
   render() {
-    const listItems = [...Array(this.props.pages).keys()].map(p => p + 1).map((p) => {
-        const isCurrent = (p === this.props.page);
+    const currentPageInt = parseInt(this.props.page);
+    const totalPagesInt = parseInt(this.props.pages);
+    const siblingCount = 3;
+    const totalPageNumbers = siblingCount + 5;
+    const paginationRange = this.getPaginationRange(currentPageInt, totalPagesInt, siblingCount, totalPageNumbers);
+
+    let dotCount = 0;
+    const shownItems = paginationRange.map(p => {
+        if (p === DOTS) {
+          dotCount++;
+          return <Pagination.Ellipsis key={`dot-${dotCount}`} />
+        }
+
+        const isCurrent = (p === currentPageInt);
         const props = {
           active: isCurrent,
           onClick: e => this.handlePageChange(e, p)
         };
         return <Pagination.Item key={p} href={authors_path(this.props.letter, p)} {...props}>{p}</Pagination.Item>
-      }
-    );
+    });
+
     const props = p => {
       const isValid = (p > 0 && p <= this.props.pages);
       return { 
@@ -28,16 +88,35 @@ export default class NumberPagination extends Component {
       }
     };
 
+    const getDropdown = () => {
+      return (
+        <Dropdown className="page-item">
+          <Dropdown.Toggle className="page-link" id="dropdown-basic-button">Jump to page...</Dropdown.Toggle>
+          <Dropdown.Menu>
+            {[...Array(totalPagesInt).keys()].map(p => p + 1).map((p) => {
+              if (p !== currentPageInt) {
+                return <Dropdown.Item key={`${p}-link`}
+                                      onClick={e => this.handlePageChange(e, p)}>{p}</Dropdown.Item>
+              }
+            })}
+          </Dropdown.Menu>
+        </Dropdown>
+      )
+    }
+    const dropdown = (totalPageNumbers >= totalPagesInt) ? "" : getDropdown();
+
     return (
       <nav aria-label="Page labels">
         <div className="text-center">
           <Pagination className="justify-content-center">
-            <Pagination.Prev {...props(this.props.page - 1)} />
-            {listItems}
-            <Pagination.Next {...props(parseInt(this.props.page) + 1)} />
+            <Pagination.Prev {...props(currentPageInt - 1)} />
+            {shownItems}
+            <Pagination.Next {...props(currentPageInt + 1)} />
+            {dropdown}
           </Pagination>
         </div>
-      </nav>)
+      </nav>
+    )
   }
 }