Merge pull request #54 from diging/develop

Story/geco 90 (#53)
diging · Feb 16, 2018 · d042d1d · d042d1d
2 parents 9ff16e7 + 4cd6d94
commit d042d1d
Show file tree

Hide file tree

Showing 62 changed files with 18,829 additions and 44 deletions.
diff --git a/giles-eco/pom.xml b/giles-eco/pom.xml
@@ -17,7 +17,7 @@
         <org.slf4j-version>1.7.5</org.slf4j-version>
         <com.fasterxml.jackson.version>2.8.4</com.fasterxml.jackson.version>
 
-        <geco.requests.version>0.5</geco.requests.version>
+        <geco.requests.version>0.6</geco.requests.version>
         <geco.util.version>0.4.3</geco.util.version>
         <geco.september.util.version>0.2</geco.september.util.version>
 
@@ -59,7 +59,7 @@
         <deploy.path.prod>giles</deploy.path.prod>
         <pullrequest></pullrequest>
         <buildNumber></buildNumber>
-        <log.level>debug</log.level>
+        <log.level>info</log.level>
         <iframing.hostname></iframing.hostname>
 
         <!-- DB config -->
@@ -388,11 +388,11 @@
             <version>2.4</version>
         </dependency>
 
-        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons- 3 -->
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
-            <version>3.4</version>
+            <version>3.7</version>
         </dependency>
 
         <!-- PDF library -->

diff --git a/...o/src/main/java/edu/asu/diging/gilesecosystem/web/controllers/ViewDocumentController.java b/...o/src/main/java/edu/asu/diging/gilesecosystem/web/controllers/ViewDocumentController.java
@@ -139,6 +139,9 @@ public void accept(IProcessingRequest t) {
 
     private void setRequestStatus(FilePageBean bean, Map<String, List<IProcessingRequest>> requestsByFileId) {
         List<IProcessingRequest> fileReqs = requestsByFileId.get(bean.getId());
+        if (fileReqs == null) {
+            return;
+        }
         for (IProcessingRequest req : fileReqs) {
             if (req.getSentRequest() instanceof StorageRequest) {
                 bean.setStoredStatus(req.getRequestStatus());

diff --git a/...s-eco/src/main/java/edu/asu/diging/gilesecosystem/web/controllers/pages/PagePageBean.java b/...s-eco/src/main/java/edu/asu/diging/gilesecosystem/web/controllers/pages/PagePageBean.java
@@ -3,6 +3,7 @@
 import edu.asu.diging.gilesecosystem.web.domain.IDocument;
 import edu.asu.diging.gilesecosystem.web.domain.IFile;
 import edu.asu.diging.gilesecosystem.web.domain.IPage;
+import edu.asu.diging.gilesecosystem.web.domain.PageStatus;
 
 public class PagePageBean implements IPage {
 
@@ -11,6 +12,32 @@ public class PagePageBean implements IPage {
     private String textFileId;
     private String ocrFileId;
 
+    private PageStatus imageFileStatus;
+    private PageStatus textFileStatus;
+    private PageStatus ocrFileStatus;
+
+    private String imageFileErrorMsg;
+    public String getImageFileErrorMsg() {
+        return imageFileErrorMsg;
+    }
+    public void setImageFileErrorMsg(String imageFileErrorMsg) {
+        this.imageFileErrorMsg = imageFileErrorMsg;
+    }
+    public String getTextFileErrorMsg() {
+        return textFileErrorMsg;
+    }
+    public void setTextFileErrorMsg(String textFileErrorMsg) {
+        this.textFileErrorMsg = textFileErrorMsg;
+    }
+    public String getOcrFileErrorMsg() {
+        return ocrFileErrorMsg;
+    }
+    public void setOcrFileErrorMsg(String ocrFileErrorMsg) {
+        this.ocrFileErrorMsg = ocrFileErrorMsg;
+    }
+    private String textFileErrorMsg;
+    private String ocrFileErrorMsg;
+
     private IFile imageFile;
     private IFile textFile;
     private IFile ocrFile;
@@ -93,5 +120,23 @@ public IDocument getDocument() {
         // TODO Auto-generated method stub
         return null;
     }
+    public PageStatus getImageFileStatus() {
+        return imageFileStatus;
+    }
+    public void setImageFileStatus(PageStatus imageFileStatus) {
+        this.imageFileStatus = imageFileStatus;
+    }
+    public PageStatus getTextFileStatus() {
+        return textFileStatus;
+    }
+    public void setTextFileStatus(PageStatus textFileStatus) {
+        this.textFileStatus = textFileStatus;
+    }
+    public PageStatus getOcrFileStatus() {
+        return ocrFileStatus;
+    }
+    public void setOcrFileStatus(PageStatus ocrFileStatus) {
+        this.ocrFileStatus = ocrFileStatus;
+    }
 
 }
diff --git a/giles-eco/src/main/java/edu/asu/diging/gilesecosystem/web/domain/IPage.java b/giles-eco/src/main/java/edu/asu/diging/gilesecosystem/web/domain/IPage.java
@@ -23,4 +23,28 @@ public interface IPage {
 
     IDocument getDocument();
 
+    void setOcrFileStatus(PageStatus ocrFileStatus);
+
+    PageStatus getOcrFileStatus();
+
+    void setTextFileStatus(PageStatus textFileStatus);
+
+    PageStatus getTextFileStatus();
+
+    void setImageFileStatus(PageStatus imageFileStatus);
+
+    PageStatus getImageFileStatus();
+
+    void setTextFileErrorMsg(String textFileErrorMsg);
+
+    String getTextFileErrorMsg();
+
+    void setOcrFileErrorMsg(String ocrFileErrorMsg);
+
+    String getOcrFileErrorMsg();
+
+    void setImageFileErrorMsg(String imageFileErrorMsg);
+
+    String getImageFileErrorMsg();
+
 }
diff --git a/giles-eco/src/main/java/edu/asu/diging/gilesecosystem/web/domain/PageStatus.java b/giles-eco/src/main/java/edu/asu/diging/gilesecosystem/web/domain/PageStatus.java
@@ -0,0 +1,17 @@
+package edu.asu.diging.gilesecosystem.web.domain;
+
+
+
+/**
+ * Status enum to indicate if a page was successfully processed. 
+ * 
+ * Note: if {@link edu.asu.diging.gilesecosystem.requests.PageStatus} changes, this enum has
+ * to be adjusted to map statuses correctly using the valueof() method.
+ * 
+ * @author jdamerow
+ *
+ */
+public enum PageStatus {
+    COMPLETE,
+    FAILED;
+}
diff --git a/giles-eco/src/main/java/edu/asu/diging/gilesecosystem/web/domain/impl/Page.java b/giles-eco/src/main/java/edu/asu/diging/gilesecosystem/web/domain/impl/Page.java
@@ -1,6 +1,8 @@
 package edu.asu.diging.gilesecosystem.web.domain.impl;
 
 import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.persistence.FetchType;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
@@ -10,6 +12,7 @@
 
 import edu.asu.diging.gilesecosystem.web.domain.IDocument;
 import edu.asu.diging.gilesecosystem.web.domain.IPage;
+import edu.asu.diging.gilesecosystem.web.domain.PageStatus;
 
 @Entity
 public class Page implements IPage {
@@ -20,6 +23,18 @@ public class Page implements IPage {
     private String textFileId;
     private String ocrFileId;
 
+    @Enumerated(EnumType.STRING)
+    private PageStatus imageFileStatus;
+    private String imageFileErrorMsg;
+
+    @Enumerated(EnumType.STRING)
+    private PageStatus textFileStatus;
+    private String textFileErrorMsg;
+
+    @Enumerated(EnumType.STRING)
+    private PageStatus ocrFileStatus;
+    private String ocrFileErrorMsg;
+
     @ManyToOne(fetch = FetchType.LAZY, targetEntity=Document.class)
     @JoinColumn(name = "document_id", nullable = false)
     private IDocument document;
@@ -88,4 +103,52 @@ public IDocument getDocument() {
     public void setDocument(IDocument document) {
         this.document = document;
     }
+    @Override
+    public PageStatus getImageFileStatus() {
+        return imageFileStatus;
+    }
+    @Override
+    public void setImageFileStatus(PageStatus imageFileStatus) {
+        this.imageFileStatus = imageFileStatus;
+    }
+    @Override
+    public PageStatus getTextFileStatus() {
+        return textFileStatus;
+    }
+    @Override
+    public void setTextFileStatus(PageStatus textFileStatus) {
+        this.textFileStatus = textFileStatus;
+    }
+    @Override
+    public PageStatus getOcrFileStatus() {
+        return ocrFileStatus;
+    }
+    @Override
+    public void setOcrFileStatus(PageStatus ocrFileStatus) {
+        this.ocrFileStatus = ocrFileStatus;
+    }
+    @Override
+    public String getImageFileErrorMsg() {
+        return imageFileErrorMsg;
+    }
+    @Override
+    public void setImageFileErrorMsg(String imageFileErrorMsg) {
+        this.imageFileErrorMsg = imageFileErrorMsg;
+    }
+    @Override
+    public String getOcrFileErrorMsg() {
+        return ocrFileErrorMsg;
+    }
+    @Override
+    public void setOcrFileErrorMsg(String ocrFileErrorMsg) {
+        this.ocrFileErrorMsg = ocrFileErrorMsg;
+    }
+    @Override
+    public String getTextFileErrorMsg() {
+        return textFileErrorMsg;
+    }
+    @Override
+    public void setTextFileErrorMsg(String textFileErrorMsg) {
+        this.textFileErrorMsg = textFileErrorMsg;
+    }
 }
diff --git a/.../asu/diging/gilesecosystem/web/service/processing/impl/ACompletedExtractionProcessor.java b/.../asu/diging/gilesecosystem/web/service/processing/impl/ACompletedExtractionProcessor.java
@@ -50,7 +50,7 @@ public abstract class ACompletedExtractionProcessor extends ACompletedRequestPro
     @Autowired
     private ISystemMessageHandler messageHandler;
 
-    protected void sendRequest(IFile file, String downloadPath, String downloadUrl, FileType type) {
+    protected void sendStorageRequest(IFile file, String downloadPath, String downloadUrl, FileType type) {
         IStorageRequest storageRequest;
         try {
             storageRequest = requestHelper.createStorageRequest(file, downloadPath, downloadUrl, type, fileService.generateRequestId(REQUEST_PREFIX));

diff --git a/.../diging/gilesecosystem/web/service/processing/impl/CompletedImageExtractionProcessor.java b/.../diging/gilesecosystem/web/service/processing/impl/CompletedImageExtractionProcessor.java
@@ -17,6 +17,7 @@
 import edu.asu.diging.gilesecosystem.web.domain.IDocument;
 import edu.asu.diging.gilesecosystem.web.domain.IFile;
 import edu.asu.diging.gilesecosystem.web.domain.IPage;
+import edu.asu.diging.gilesecosystem.web.domain.PageStatus;
 import edu.asu.diging.gilesecosystem.web.domain.ProcessingStatus;
 import edu.asu.diging.gilesecosystem.web.domain.impl.Page;
 import edu.asu.diging.gilesecosystem.web.exceptions.GilesProcessingException;
@@ -78,9 +79,14 @@ public void processRequest(ICompletedImageExtractionRequest request) {
                     documentPage.setDocument(document);
                 }
                 documentPage.setImageFileId(pageText.getId());
+                // this is a hack, ideally we would map the statuses somewhere, but well...
+                documentPage.setImageFileStatus(PageStatus.valueOf(page.getStatus().toString()));
+                documentPage.setImageFileErrorMsg(page.getErrorMsg());
 
-                sendRequest(pageText, page.getPathToFile(), page.getDownloadUrl(), FileType.IMAGE);
-            }
+                if (page.getStatus() == edu.asu.diging.gilesecosystem.requests.PageStatus.COMPLETE) {
+                    sendStorageRequest(pageText, page.getPathToFile(), page.getDownloadUrl(), FileType.IMAGE);
+                }
+           }
         } else {
             request.setStatus(RequestStatus.FAILED);
         }

diff --git a/...java/edu/asu/diging/gilesecosystem/web/service/processing/impl/CompletedOCRProcessor.java b/...java/edu/asu/diging/gilesecosystem/web/service/processing/impl/CompletedOCRProcessor.java
@@ -19,6 +19,7 @@
 import edu.asu.diging.gilesecosystem.web.domain.IDocument;
 import edu.asu.diging.gilesecosystem.web.domain.IFile;
 import edu.asu.diging.gilesecosystem.web.domain.IPage;
+import edu.asu.diging.gilesecosystem.web.domain.PageStatus;
 import edu.asu.diging.gilesecosystem.web.domain.ProcessingStatus;
 import edu.asu.diging.gilesecosystem.web.domain.impl.Page;
 import edu.asu.diging.gilesecosystem.web.exceptions.GilesProcessingException;
@@ -76,11 +77,17 @@ public void processRequest(ICompletedOCRRequest request) {
             documentPage.setDocument(document);
         }
         documentPage.setOcrFileId(pageText.getId());
+        if (request.getStatus() != null) {
+            documentPage.setOcrFileStatus(PageStatus.valueOf(request.getStatus().toString())); 
+        } else {
+            documentPage.setOcrFileStatus(PageStatus.COMPLETE);
+        }
+        documentPage.setOcrFileErrorMsg(request.getErrorMsg());
 
         if (request.getDownloadPath() != null && !request.getDownloadPath().isEmpty()
-                && request.getDownloadUrl() != null & !request.getDownloadUrl().isEmpty()) {
+                && request.getDownloadUrl() != null & !request.getDownloadUrl().isEmpty() && request.getStatus() != RequestStatus.FAILED) {
             request.setStatus(RequestStatus.COMPLETE);
-            sendRequest(pageText, request.getDownloadPath(), request.getDownloadUrl(), FileType.TEXT);
+            sendStorageRequest(pageText, request.getDownloadPath(), request.getDownloadUrl(), FileType.TEXT);
         } else {
             request.setStatus(RequestStatus.FAILED);
         }

diff --git a/...u/diging/gilesecosystem/web/service/processing/impl/CompletedTextExtractionProcessor.java b/...u/diging/gilesecosystem/web/service/processing/impl/CompletedTextExtractionProcessor.java
@@ -15,6 +15,7 @@
 import edu.asu.diging.gilesecosystem.web.domain.IDocument;
 import edu.asu.diging.gilesecosystem.web.domain.IFile;
 import edu.asu.diging.gilesecosystem.web.domain.IPage;
+import edu.asu.diging.gilesecosystem.web.domain.PageStatus;
 import edu.asu.diging.gilesecosystem.web.domain.ProcessingStatus;
 import edu.asu.diging.gilesecosystem.web.domain.impl.Page;
 import edu.asu.diging.gilesecosystem.web.exceptions.GilesProcessingException;
@@ -56,7 +57,6 @@ public void processRequest(ICompletedTextExtractionRequest request) {
         String completeTextDownload = request.getDownloadUrl();
         // text was extracted
         if (completeTextDownload != null && !completeTextDownload.isEmpty()) {
-            request.setStatus(RequestStatus.COMPLETE);
             IFile completeText = createFile(file, document, MediaType.TEXT_PLAIN_VALUE, request.getSize(), request.getTextFilename(), REQUEST_PREFIX);
 
             try {
@@ -68,10 +68,8 @@ public void processRequest(ICompletedTextExtractionRequest request) {
 
             document.setExtractedTextFileId(completeText.getId());
 
-            sendRequest(completeText, request.getDownloadPath(), request.getDownloadUrl(), FileType.TEXT);
-        } else {
-            request.setStatus(RequestStatus.FAILED);
-        }
+            sendStorageRequest(completeText, request.getDownloadPath(), request.getDownloadUrl(), FileType.TEXT);
+        } 
 
         if (request.getPages() != null ) {
             for (edu.asu.diging.gilesecosystem.requests.impl.Page page : request.getPages()) {
@@ -88,10 +86,13 @@ public void processRequest(ICompletedTextExtractionRequest request) {
                 documentPage.setDocument(document);
                 documentPage.setPageNr(page.getPageNr());
                 documentPage.setTextFileId(pageText.getId());
-
+                if (page.getStatus() != null) {
+                    documentPage.setTextFileStatus(PageStatus.valueOf(page.getStatus().toString()));
+                }
+                documentPage.setTextFileErrorMsg(page.getErrorMsg());
                 document.getPages().add(documentPage);
 
-                sendRequest(pageText, page.getPathToFile(), page.getDownloadUrl(), FileType.TEXT);
+                sendStorageRequest(pageText, page.getPathToFile(), page.getDownloadUrl(), FileType.TEXT);
             }
         }
 

diff --git a/giles-eco/src/main/resources/META-INF/hibernate/orm.xml b/giles-eco/src/main/resources/META-INF/hibernate/orm.xml
@@ -20,6 +20,7 @@
         </property>
         <property name="downloadUrl" type="string" />
         <property name="downloadPath" type="string" />
+        <property name="errorMsg" type="string" />
         <joined-subclass name="edu.asu.diging.gilesecosystem.requests.impl.CompletedImageExtractionRequest" table="CompletedImageExtractionRequest">
             <key column="id" />
             <property name="extractionDate" type="string" />
@@ -77,7 +78,7 @@
             <property name="filename" type="string" />
         </joined-subclass>
     </class>
-    <class name="edu.asu.diging.gilesecosystem.requests.impl.Page" table="RequestPage">
+    <class name="edu.asu.diging.gilesecosystem.requests.impl.Page" table="RequestPage" >
         <id name="id" column="pageId">
             <generator class="increment" />
         </id>

diff --git a/giles-eco/src/main/resources/log4j.xml b/giles-eco/src/main/resources/log4j.xml
@@ -45,7 +45,7 @@
     </logger>
 
     <logger name="org.hibernate">
-        <level value="${log.level}" />
+        <level value="info" />
     </logger>
 
 	<!-- Root Logger -->