Merge pull request #96 from IndicoDataSolutions/auto-review-docsrc

ADD: submission auto review documentation

docs/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: a5f3bfb9819a94b12018d8f758c9445f
+config: 8822370fb4ea81841d02fe47ddc19881
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_sources/auto-review.rst.txt

@@ -0,0 +1,112 @@
+Auto Review
+************
+Introduction
+=============
+Auto Review is a feature that allows users to programatically accept and reject predictions before Review. The criteria for accepting and rejecting predictions can be customized by the user. 
+
+To use Auto-review, Review and Scripted Review must be enabled from the workflows page. You can enable these features by clicking on workflows from your dataset page (shown below).
+
+.. image:: enable_auto_review.png
+
+Using Auto Review
+=================
+Auto review is currently only available via api call. To use it, the following steps must be performed:
+
+- Submit a document through your workflow
+- Retreive the submission
+- Add a flag on each prediction result you wish to Auto Review in the submission to designate if you would like to accept or reject it
+- Submit the changes to Review
+
+Example usage of Auto Review::
+
+    from indico.queries import (
+        WorkflowSubmission,
+        WaitForSubmissions,
+        RetrieveStorageObject,
+        SubmitReview
+    )
+    # Submit a document to workflow
+    submission_ids = client.call(
+        WorkflowSubmission(
+            workflow_id=workflow_id, files=["my_file.pdf"]
+        )
+    ) 
+    # Retreive the submission results
+    submissions = client.call(WaitForSubmissions(submission_ids))
+    submission = submissions[0]
+    raw_result = client.call(RetrieveStorageObject(submission.result_file))
+    changes = raw_result["results"]["document"]["results"]
+    # Add flags to reject or accept predictions
+    for model, preds in changes.items():
+        for pred in preds:
+            if pred["label"] == "Name" and len(pred["text"]) < 3:
+                # Reject Name labels shorter than 3
+                pred["rejected"] = True
+            else:
+                # Accept all other labels
+                pred["accepted"] = True
+    # Submit the changes
+    client.call(SubmitReview(submission.id, changes=changes))
+
+After submitting the changes using Auto-Review, the document will be available through Review for human review. Any predictions that were accepted through Auto-Review will be pre-approved on the review page (shown below). Reviewers can still modify and add to accepted predictions in Review.
+
+.. figure:: accepted_prediction.png
+    :align: right
+
+    Accepted predictions
+
+.. figure:: not_accepted_prediction.png
+    :align: left
+    :scale: 96
+
+    Normal predictions
+
+
+.. figure:: rejected_prediction.png
+    :align: right
+    :scale: 93
+
+    Rejected prediction
+
+Predictions that were rejected through Auto-Review will not appear in Review (shown right). Human reviewers are still able to fill in these empty labels, or confirm no value. You can also confirm no value programmatically through Auto Review.
+
+
+Auto Reject a Document
+======================
+When submitting changes for a given document using SubmitReview, you can reject a document and send it to the exceptions queue using the "rejected" argument::
+
+    client.call(SubmitReview(submission.id, changes=changes, rejected=True))
+
+Accepting No Value
+==================
+Even when all predictions are accepted, a reviewer will still need to manually accept any labels that are empty. However, you can automatically accept empty values for specific lables using the "_no_value" key. Within the model results, set the dictionary key "_no_value" equal to a dictionary containing the name of your model keying to a list of labels. This list of labels should contain all the labels you would like to automatically accept empty values for. Accepted empty labels will appear in review as accepted labels. Please note that you can only auto-accept empty labels that do not contain any predictions. Labels that contain predictions should not be inlcuded in the list of no value labels. In the example below empty labels are only accepted if the specified label is not present in the predictions::
+
+    from indico.queries import (
+        WorkflowSubmission,
+        WaitForSubmissions,
+        RetrieveStorageObject,
+        SubmitReview
+    )
+    # Submit a document to workflow
+    submission_ids = client.call(
+        WorkflowSubmission(
+            workflow_id=workflow_id, files=["my_file.pdf"]
+        )
+    ) 
+    # Retreive the submission results
+    submissions = client.call(WaitForSubmissions(submission_ids))
+    submission = submissions[0]
+    raw_result = client.call(RetrieveStorageObject(submission.result_file))
+    changes = raw_result["results"]["document"]["results"]
+    # Accept empty values for Name label
+    for model, preds in changes.copy().items():
+        pred_labels = set()
+        for pred in preds:
+            pred_labels.add(pred["label"])
+        if "Name" not in pred_labels:
+            changes["_no_value"] = {model: ["Name"]}
+    # Submit the changes
+    client.call(SubmitReview(submission.id, changes=changes))
+
+
+

docs/_sources/index.rst.txt

@@ -15,6 +15,7 @@ User Guide
     model_predictions
     workflow-intro
     review-intro
+    auto-review
     object-detection
     migration
     examples
@@ -48,4 +49,4 @@ Reference
     dataset-type
     jobs-type
     model-group-type
-    model-type
+    model-type

docs/_static/basic.css

@@ -4,7 +4,7 @@
  *
  * Sphinx stylesheet -- basic theme.
  *
- * :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */
@@ -693,7 +693,7 @@ pre {
     overflow-y: hidden;  /* fixes display issues on Chrome browsers */
 }
 
-pre, div[class|="highlight"] {
+pre, div[class*="highlight-"] {
     clear: both;
 }
 
@@ -704,7 +704,7 @@ span.pre {
     hyphens: none;
 }
 
-div[class^="highlight-"] {
+div[class*="highlight-"] {
     margin: 1em 0;
 }
 
@@ -764,6 +764,7 @@ div.code-block-caption code {
 }
 
 table.highlighttable td.linenos,
+span.linenos,
 div.doctest > div.highlight span.gp {  /* gp: Generic.Prompt */
     user-select: none;
 }

docs/_static/doctools.js

@@ -4,7 +4,7 @@
  *
  * Sphinx JavaScript utilities for all documentation.
  *
- * :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */
@@ -285,9 +285,10 @@ var Documentation = {
   initOnKeyListeners: function() {
     $(document).keydown(function(event) {
       var activeElementType = document.activeElement.tagName;
-      // don't navigate when in search box or textarea
+      // don't navigate when in search box, textarea, dropdown or button
       if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT'
-          && !event.altKey && !event.ctrlKey && !event.metaKey && !event.shiftKey) {
+          && activeElementType !== 'BUTTON' && !event.altKey && !event.ctrlKey && !event.metaKey
+          && !event.shiftKey) {
         switch (event.keyCode) {
           case 37: // left
             var prevHref = $('link[rel="prev"]').prop('href');

docs/_static/language_data.js

@@ -5,7 +5,7 @@
  * This script contains the language-specific data used by searchtools.js,
  * namely the list of stopwords, stemmer, scorer and splitter.
  *
- * :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */

docs/_static/pygments.css

@@ -1,5 +1,10 @@
+pre { line-height: 125%; }
+td.linenos pre { color: #000000; background-color: #f0f0f0; padding-left: 5px; padding-right: 5px; }
+span.linenos { color: #000000; background-color: #f0f0f0; padding-left: 5px; padding-right: 5px; }
+td.linenos pre.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
+span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
 .highlight .hll { background-color: #ffffcc }
-.highlight  { background: #f8f8f8; }
+.highlight { background: #f8f8f8; }
 .highlight .c { color: #408080; font-style: italic } /* Comment */
 .highlight .err { border: 1px solid #FF0000 } /* Error */
 .highlight .k { color: #008000; font-weight: bold } /* Keyword */

docs/_static/searchtools.js

@@ -4,7 +4,7 @@
  *
  * Sphinx JavaScript utilities for the full-text search.
  *
- * :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */
@@ -59,10 +59,10 @@ var Search = {
   _pulse_status : -1,
 
   htmlToText : function(htmlString) {
-      var htmlElement = document.createElement('span');
-      htmlElement.innerHTML = htmlString;
-      $(htmlElement).find('.headerlink').remove();
-      docContent = $(htmlElement).find('[role=main]')[0];
+      var virtualDocument = document.implementation.createHTMLDocument('virtual');
+      var htmlElement = $(htmlString, virtualDocument);
+      htmlElement.find('.headerlink').remove();
+      docContent = htmlElement.find('[role=main]')[0];
       if(docContent === undefined) {
           console.warn("Content block not found. Sphinx search tries to obtain it " +
                        "via '[role=main]'. Could you check your theme or template.");
@@ -166,8 +166,7 @@ var Search = {
           objectterms.push(tmp[i].toLowerCase());
       }
 
-      if ($u.indexOf(stopwords, tmp[i].toLowerCase()) != -1 || tmp[i].match(/^\d+$/) ||
-          tmp[i] === "") {
+      if ($u.indexOf(stopwords, tmp[i].toLowerCase()) != -1 || tmp[i] === "") {
         // skip this "word"
         continue;
       }