update submission filter (#294)

nateshim-indico · Nathanael Shim · web-flow · commit 1547bd646aa1 · 2024-03-05T09:07:30.000-08:00
* update submission filter

* update submission query

* update list submissions

* add tests

* resolve tests

* add docstrings

* add test case for input_filename

* address comments

* fix docstring edit location

* add codeowners file

* Address comments

* fix typing in filter init

---------

Co-authored-by: Nathanael Shim &lt;nathanaelshim@Nathanaels-MacBook-Pro.local&gt;
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -0,0 +1 @@
+* @IndicoDataSolutions/pr-be-indicodata-ai
diff --git a/examples/submission-filters.py b/examples/submission-filters.py
@@ -0,0 +1,52 @@
+from indico import IndicoClient, IndicoConfig
+from indico.filters import DateRangeFilter, SubmissionFilter, and_, or_
+from indico.queries import ListSubmissions
+
+# Create an Indico API client
+my_config = IndicoConfig(
+    host="try.indico.io", api_token_path="./path/to/indico_api_token.txt"
+)
+client = IndicoClient(config=my_config)
+
+workflow_id = 5
+
+"""
+Example 1
+List all submissions that are COMPLETE or FAILED
+"""
+sub_filter = or_(SubmissionFilter(status="COMPLETE"), SubmissionFilter(status="FAILED"))
+submissions = client.call(ListSubmissions(filters=sub_filter))
+
+"""
+Example 2
+List all submissions that are COMPLETE and FAILED
+"""
+sub_filter = and_(
+    SubmissionFilter(status="COMPLETE"), SubmissionFilter(status="FAILED")
+)
+submisions = client.call(ListSubmissions(filters=sub_filter))
+
+"""
+Example 3
+List all submissions that are retrieved and have a filename that contains 'property'
+"""
+sub_filter = SubmissionFilter(retrieved=True, input_filename="property")
+submissions = client.call(ListSubmissions(filters=sub_filter))
+
+"""
+Example 4
+List all submissions that are created and updated within a certain date range
+"""
+date_filter = DateRangeFilter(filter_from="2022-01-01", filter_to="2023-01-01")
+sub_filter = SubmissionFilter(created_at=date_filter, updated_at=date_filter)
+submissions = client.call(ListSubmissions(filters=sub_filter))
+
+"""
+Example 5
+List all submissions that are not in progress of being reviewed and are completed
+"""
+submissions = client.call(
+    ListSubmissions(
+        filters=SubmissionFilter(status="COMPLETE", review_in_progress=False)
+    )
+)
diff --git a/indico/filters/__init__.py b/indico/filters/__init__.py
@@ -1,5 +1,5 @@
 import datetime
-from typing import Any, Iterable, Mapping
+from typing import Any, Iterable, List, Mapping, Union
 
 from indico.errors import IndicoInputError
 
@@ -48,7 +48,10 @@ class SubmissionReviewFilter(Filter):
     __options__ = ("rejected", "created_by", "review_type")
 
     def __init__(
-        self, rejected: bool = None, created_by: int = None, review_type: str = None
+        self,
+        rejected: Union[bool, None] = None,
+        created_by: Union[int, None] = None,
+        review_type: Union[str, None] = None,
     ):
         kwargs = {
             "rejected": rejected,
@@ -59,33 +62,75 @@ def __init__(
         super().__init__(**kwargs)
 
 
+class DateRangeFilter(dict):
+    """
+    Create a Filter when querying for Submissions within a certain date range
+    Args:
+        filter_from (str): A valid string representation of a datetime for start date to filter
+        filter_to (str): A valid string representation of a datetime for end date to filter
+    """
+
+    def __init__(
+        self, filter_from: Union[str, None] = None, filter_to: Union[str, None] = None
+    ):
+        kwargs = {"from": filter_from, "to": filter_to}
+        self.update(kwargs)
+
+
 class SubmissionFilter(Filter):
     """
     Create a Filter when querying for WorkflowSubmissions.
 
     Args:
+        file_type (list): submissions with a file type in this list. Options:
+            [CSV, PDF, EXCEL, DOC, DOCX, PPT, PPTX, PNG, JPG, TIFF, TXT, RTF, XLS, XLSX, UNKNOWN, MSG, EML]
         input_filename (str): submissions with input file names containing this string
         status (str): submissions in this status. Options:
             [PROCESSING, PENDING_REVIEW, PENDING_ADMIN_REVIEW, COMPLETE, FAILED]
-        retrieved(bool): Filter submissions on the retrieved flag
+        retrieved (bool): submissions that have been retrieved (True) or not (False)
+        reviews (SubmissionReviewFilter): submissions whose completed reviews match this review filter
+        review_in_progress (bool): submissions where a review is in progress (True) or not (False)
+        files_deleted (bool): submissions that have had their internal files removed (True) or not (False)
+        created_at (DateRangeFilter): submissions created during given time range
+        updated_at (DateRangeFilter): submissions updated during given time range
     Returns:
         dict containing query filter parameters
     """
 
-    __options__ = ("input_filename", "status", "retrieved")
+    __options__ = (
+        "file_type",
+        "input_filename",
+        "status",
+        "retrieved",
+        "reviews",
+        "review_in_progress",
+        "files_deleted",
+        "created_at",
+        "updated_at",
+    )
 
     def __init__(
         self,
-        input_filename: str = None,
-        status: str = None,
-        retrieved: bool = None,
-        reviews: SubmissionReviewFilter = None,
+        file_type: Union[List[str], None] = None,
+        input_filename: Union[str, None] = None,
+        status: Union[str, None] = None,
+        retrieved: Union[bool, None] = None,
+        reviews: Union[SubmissionReviewFilter, None] = None,
+        review_in_progress: Union[bool, None] = None,
+        files_deleted: Union[bool, None] = None,
+        created_at: Union[DateRangeFilter, None] = None,
+        updated_at: Union[DateRangeFilter, None] = None,
     ):
         kwargs = {
+            "filetype": file_type,
             "inputFilename": input_filename,
             "status": status.upper() if status else status,
             "retrieved": retrieved,
             "reviews": reviews,
+            "reviewInProgress": review_in_progress,
+            "filesDeleted": files_deleted,
+            "createdAt": created_at,
+            "updatedAt": updated_at,
         }
 
         super().__init__(**kwargs)
@@ -110,10 +155,10 @@ class ModelGroupExampleFilter(Filter):
 
     def __init__(
         self,
-        file_name: str = None,
-        partial: bool = None,
-        status: str = None,
-        text_search: str = None,
+        file_name: Union[str, None] = None,
+        partial: Union[bool, None] = None,
+        status: Union[str, None] = None,
+        text_search: Union[str, None] = None,
     ):
         kwargs = {
             "fileName": file_name,
@@ -138,7 +183,9 @@ class UserMetricsFilter(Filter):
 
     __options__ = ("user_id", "user_email")
 
-    def __init__(self, user_id: int = None, user_email: str = None):
+    def __init__(
+        self, user_id: Union[int, None] = None, user_email: Union[str, None] = None
+    ):
         kwargs = {"userId": user_id, "userEmail": user_email}
 
         super().__init__(**kwargs)
@@ -172,32 +219,36 @@ class DocumentReportFilter(Filter):
 
     def __init__(
         self,
-        submission_id: int = None,
-        workflow_id: int = None,
-        status: str = None,
-        created_at_start_date: datetime = None,
-        created_at_end_date: datetime = None,
-        updated_at_start_date: datetime = None,
-        updated_at_end_date: datetime = None,
+        submission_id: Union[int, None] = None,
+        workflow_id: Union[int, None] = None,
+        status: Union[str, None] = None,
+        created_at_start_date: Union[datetime.datetime, None] = None,
+        created_at_end_date: Union[datetime.datetime, None] = None,
+        updated_at_start_date: Union[datetime.datetime, None] = None,
+        updated_at_end_date: Union[datetime.datetime, None] = None,
     ):
         kwargs = {"workflowId": workflow_id, "id": submission_id, "status": status}
         if created_at_end_date and not created_at_start_date:
             raise IndicoInputError("Must specify created_at_start_date")
         if created_at_start_date:
             kwargs["createdAt"] = {
                 "from": created_at_start_date.strftime("%Y-%m-%d"),
-                "to": created_at_end_date.strftime("%Y-%m-%d")
-                if created_at_end_date is not None
-                else datetime.datetime.now().strftime("%Y-%m-%d"),
+                "to": (
+                    created_at_end_date.strftime("%Y-%m-%d")
+                    if created_at_end_date is not None
+                    else datetime.datetime.now().strftime("%Y-%m-%d")
+                ),
             }
 
         if updated_at_end_date and not updated_at_start_date:
             raise IndicoInputError("Must specify updated_at_start_date")
         if updated_at_start_date is not None:
             kwargs["updatedAt"] = {
                 "from": updated_at_start_date.strftime("%Y-%m-%d"),
-                "to": updated_at_end_date.strftime("%Y-%m-%d")
-                if updated_at_end_date is not None
-                else datetime.datetime.now().strftime("%Y-%m-%d"),
+                "to": (
+                    updated_at_end_date.strftime("%Y-%m-%d")
+                    if updated_at_end_date is not None
+                    else datetime.datetime.now().strftime("%Y-%m-%d")
+                ),
             }
         super().__init__(**kwargs)
diff --git a/indico/queries/submission.py b/indico/queries/submission.py
@@ -33,57 +33,91 @@ class ListSubmissions(PagedRequest):
 
     query = """
         query ListSubmissions(
-            $submissionIds: [Int],
-            $workflowIds: [Int],
-            $filters: SubmissionFilter,
-            $limit: Int,
-            $orderBy: SUBMISSION_COLUMN_ENUM,
-            $desc: Boolean,
+            $submissionIds: [Int]
+            $workflowIds: [Int]
+            $filters: SubmissionFilter
+            $limit: Int
+            $orderBy: SUBMISSION_COLUMN_ENUM
+            $desc: Boolean
             $after: Int
-
-        ){
+            ) {
             submissions(
-                submissionIds: $submissionIds,
-                workflowIds: $workflowIds,
-                filters: $filters,
+                submissionIds: $submissionIds
+                workflowIds: $workflowIds
+                filters: $filters
                 limit: $limit
-                orderBy: $orderBy,
-                desc: $desc,
+                orderBy: $orderBy
+                desc: $desc
                 after: $after
-
-            ){
+            ) {
                 submissions {
+                id
+                datasetId
+                workflowId
+                status
+                createdAt
+                updatedAt
+                createdBy
+                updatedBy
+                completedAt
+                errors
+                filesDeleted
+                inputFiles {
                     id
-                    datasetId
-                    workflowId
-                    status
-                    inputFiles {
-                        id
-                        filename
-                        filepath
-                        filetype
-                        fileSize
-                        numPages
-                    }
-                    inputFile
-                    inputFilename
-                    resultFile
-                    deleted
-                    retrieved
-                    errors
-                    reviews {
-                        id
-                        createdAt
-                        createdBy
-                        completedAt
-                        rejected
-                        reviewType
-                        notes
-                    }
+                    filepath
+                    filename
+                    filetype
+                    submissionId
+                    fileSize
+                    numPages
+                }
+                inputFile
+                inputFilename
+                resultFile
+                outputFiles {
+                    id
+                    filepath
+                    submissionId
+                    componentId
+                    createdAt
+                }
+                retrieved
+                autoReview {
+                    id
+                    submissionId
+                    createdAt
+                    createdBy
+                    startedAt
+                    completedAt
+                    rejected
+                    reviewType
+                    notes
+                }
+                retries {
+                    id
+                    submissionId
+                    previousErrors
+                    previousStatus
+                    retryErrors
+                }
+                reviews {
+                    id
+                    submissionId
+                    createdAt
+                    createdBy
+                    startedAt
+                    completedAt
+                    rejected
+                    reviewType
+                    notes
+                }
+                reviewInProgress
                 }
                 pageInfo {
-                    endCursor
-                    hasNextPage
+                startCursor
+                endCursor
+                hasNextPage
+                aggregateCount
                 }
             }
         }
diff --git a/indico/types/__init__.py b/indico/types/__init__.py
@@ -3,6 +3,7 @@
 from .jobs import *
 from .model_group import *
 from .model import *
+from .output_file import *
 from .submission_file import *
 from .submission import *
 from .workflow import *
diff --git a/indico/types/base.py b/indico/types/base.py
@@ -44,7 +44,7 @@ def __init__(self, **kwargs):
             k = cc_to_snake(k)
             if k in attrs:
                 attr_type = attrs[k]
-                if inspect.isclass(attr_type) and issubclass(attr_type, BaseType):
+                if v is not None and inspect.isclass(attr_type) and issubclass(attr_type, BaseType):
                     v = attrs[k](**v)
 
                 if attr_type == JSONType:
diff --git a/indico/types/output_file.py b/indico/types/output_file.py
diff --git a/indico/types/submission.py b/indico/types/submission.py
diff --git a/tests/integration/queries/test_submission.py b/tests/integration/queries/test_submission.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+* @IndicoDataSolutions/pr-be-indicodata-ai`