Skip to content

Commit

Permalink
update submission filter (#294)
Browse files Browse the repository at this point in the history 
* update submission filter

* update submission query

* update list submissions

* add tests

* resolve tests

* add docstrings

* add test case for input_filename

* address comments

* fix docstring edit location

* add codeowners file

* Address comments

* fix typing in filter init

---------

Co-authored-by: Nathanael Shim <[email protected]>
  • Loading branch information
@nateshim-indico
nateshim-indico and Nathanael Shim authored Mar 5, 2024
1 parent dc5e982 commit 1547bd6
Show file tree
Hide file tree
Showing 9 changed files with 331 additions and 81 deletions.
1 change: 1 addition & 0 deletions CODEOWNERS
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* @IndicoDataSolutions/pr-be-indicodata-ai
52 changes: 52 additions & 0 deletions examples/submission-filters.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
from indico import IndicoClient, IndicoConfig
from indico.filters import DateRangeFilter, SubmissionFilter, and_, or_
from indico.queries import ListSubmissions

# Create an Indico API client
my_config = IndicoConfig(
host="try.indico.io", api_token_path="./path/to/indico_api_token.txt"
)
client = IndicoClient(config=my_config)

workflow_id = 5

"""
Example 1
List all submissions that are COMPLETE or FAILED
"""
sub_filter = or_(SubmissionFilter(status="COMPLETE"), SubmissionFilter(status="FAILED"))
submissions = client.call(ListSubmissions(filters=sub_filter))

"""
Example 2
List all submissions that are COMPLETE and FAILED
"""
sub_filter = and_(
SubmissionFilter(status="COMPLETE"), SubmissionFilter(status="FAILED")
)
submisions = client.call(ListSubmissions(filters=sub_filter))

"""
Example 3
List all submissions that are retrieved and have a filename that contains 'property'
"""
sub_filter = SubmissionFilter(retrieved=True, input_filename="property")
submissions = client.call(ListSubmissions(filters=sub_filter))

"""
Example 4
List all submissions that are created and updated within a certain date range
"""
date_filter = DateRangeFilter(filter_from="2022-01-01", filter_to="2023-01-01")
sub_filter = SubmissionFilter(created_at=date_filter, updated_at=date_filter)
submissions = client.call(ListSubmissions(filters=sub_filter))

"""
Example 5
List all submissions that are not in progress of being reviewed and are completed
"""
submissions = client.call(
ListSubmissions(
filters=SubmissionFilter(status="COMPLETE", review_in_progress=False)
)
)
103 changes: 77 additions & 26 deletions indico/filters/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import datetime
from typing import Any, Iterable, Mapping
from typing import Any, Iterable, List, Mapping, Union

from indico.errors import IndicoInputError

Expand Down Expand Up @@ -48,7 +48,10 @@ class SubmissionReviewFilter(Filter):
__options__ = ("rejected", "created_by", "review_type")

def __init__(
self, rejected: bool = None, created_by: int = None, review_type: str = None
self,
rejected: Union[bool, None] = None,
created_by: Union[int, None] = None,
review_type: Union[str, None] = None,
):
kwargs = {
"rejected": rejected,
Expand All @@ -59,33 +62,75 @@ def __init__(
super().__init__(**kwargs)


class DateRangeFilter(dict):
"""
Create a Filter when querying for Submissions within a certain date range
Args:
filter_from (str): A valid string representation of a datetime for start date to filter
filter_to (str): A valid string representation of a datetime for end date to filter
"""

def __init__(
self, filter_from: Union[str, None] = None, filter_to: Union[str, None] = None
):
kwargs = {"from": filter_from, "to": filter_to}
self.update(kwargs)


class SubmissionFilter(Filter):
"""
Create a Filter when querying for WorkflowSubmissions.
Args:
file_type (list): submissions with a file type in this list. Options:
[CSV, PDF, EXCEL, DOC, DOCX, PPT, PPTX, PNG, JPG, TIFF, TXT, RTF, XLS, XLSX, UNKNOWN, MSG, EML]
input_filename (str): submissions with input file names containing this string
status (str): submissions in this status. Options:
[PROCESSING, PENDING_REVIEW, PENDING_ADMIN_REVIEW, COMPLETE, FAILED]
retrieved(bool): Filter submissions on the retrieved flag
retrieved (bool): submissions that have been retrieved (True) or not (False)
reviews (SubmissionReviewFilter): submissions whose completed reviews match this review filter
review_in_progress (bool): submissions where a review is in progress (True) or not (False)
files_deleted (bool): submissions that have had their internal files removed (True) or not (False)
created_at (DateRangeFilter): submissions created during given time range
updated_at (DateRangeFilter): submissions updated during given time range
Returns:
dict containing query filter parameters
"""

__options__ = ("input_filename", "status", "retrieved")
__options__ = (
"file_type",
"input_filename",
"status",
"retrieved",
"reviews",
"review_in_progress",
"files_deleted",
"created_at",
"updated_at",
)

def __init__(
self,
input_filename: str = None,
status: str = None,
retrieved: bool = None,
reviews: SubmissionReviewFilter = None,
file_type: Union[List[str], None] = None,
input_filename: Union[str, None] = None,
status: Union[str, None] = None,
retrieved: Union[bool, None] = None,
reviews: Union[SubmissionReviewFilter, None] = None,
review_in_progress: Union[bool, None] = None,
files_deleted: Union[bool, None] = None,
created_at: Union[DateRangeFilter, None] = None,
updated_at: Union[DateRangeFilter, None] = None,
):
kwargs = {
"filetype": file_type,
"inputFilename": input_filename,
"status": status.upper() if status else status,
"retrieved": retrieved,
"reviews": reviews,
"reviewInProgress": review_in_progress,
"filesDeleted": files_deleted,
"createdAt": created_at,
"updatedAt": updated_at,
}

super().__init__(**kwargs)
Expand All @@ -110,10 +155,10 @@ class ModelGroupExampleFilter(Filter):

def __init__(
self,
file_name: str = None,
partial: bool = None,
status: str = None,
text_search: str = None,
file_name: Union[str, None] = None,
partial: Union[bool, None] = None,
status: Union[str, None] = None,
text_search: Union[str, None] = None,
):
kwargs = {
"fileName": file_name,
Expand All @@ -138,7 +183,9 @@ class UserMetricsFilter(Filter):

__options__ = ("user_id", "user_email")

def __init__(self, user_id: int = None, user_email: str = None):
def __init__(
self, user_id: Union[int, None] = None, user_email: Union[str, None] = None
):
kwargs = {"userId": user_id, "userEmail": user_email}

super().__init__(**kwargs)
Expand Down Expand Up @@ -172,32 +219,36 @@ class DocumentReportFilter(Filter):

def __init__(
self,
submission_id: int = None,
workflow_id: int = None,
status: str = None,
created_at_start_date: datetime = None,
created_at_end_date: datetime = None,
updated_at_start_date: datetime = None,
updated_at_end_date: datetime = None,
submission_id: Union[int, None] = None,
workflow_id: Union[int, None] = None,
status: Union[str, None] = None,
created_at_start_date: Union[datetime.datetime, None] = None,
created_at_end_date: Union[datetime.datetime, None] = None,
updated_at_start_date: Union[datetime.datetime, None] = None,
updated_at_end_date: Union[datetime.datetime, None] = None,
):
kwargs = {"workflowId": workflow_id, "id": submission_id, "status": status}
if created_at_end_date and not created_at_start_date:
raise IndicoInputError("Must specify created_at_start_date")
if created_at_start_date:
kwargs["createdAt"] = {
"from": created_at_start_date.strftime("%Y-%m-%d"),
"to": created_at_end_date.strftime("%Y-%m-%d")
if created_at_end_date is not None
else datetime.datetime.now().strftime("%Y-%m-%d"),
"to": (
created_at_end_date.strftime("%Y-%m-%d")
if created_at_end_date is not None
else datetime.datetime.now().strftime("%Y-%m-%d")
),
}

if updated_at_end_date and not updated_at_start_date:
raise IndicoInputError("Must specify updated_at_start_date")
if updated_at_start_date is not None:
kwargs["updatedAt"] = {
"from": updated_at_start_date.strftime("%Y-%m-%d"),
"to": updated_at_end_date.strftime("%Y-%m-%d")
if updated_at_end_date is not None
else datetime.datetime.now().strftime("%Y-%m-%d"),
"to": (
updated_at_end_date.strftime("%Y-%m-%d")
if updated_at_end_date is not None
else datetime.datetime.now().strftime("%Y-%m-%d")
),
}
super().__init__(**kwargs)
120 changes: 77 additions & 43 deletions indico/queries/submission.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,57 +33,91 @@ class ListSubmissions(PagedRequest):

query = """
query ListSubmissions(
$submissionIds: [Int],
$workflowIds: [Int],
$filters: SubmissionFilter,
$limit: Int,
$orderBy: SUBMISSION_COLUMN_ENUM,
$desc: Boolean,
$submissionIds: [Int]
$workflowIds: [Int]
$filters: SubmissionFilter
$limit: Int
$orderBy: SUBMISSION_COLUMN_ENUM
$desc: Boolean
$after: Int
){
) {
submissions(
submissionIds: $submissionIds,
workflowIds: $workflowIds,
filters: $filters,
submissionIds: $submissionIds
workflowIds: $workflowIds
filters: $filters
limit: $limit
orderBy: $orderBy,
desc: $desc,
orderBy: $orderBy
desc: $desc
after: $after
){
) {
submissions {
id
datasetId
workflowId
status
createdAt
updatedAt
createdBy
updatedBy
completedAt
errors
filesDeleted
inputFiles {
id
datasetId
workflowId
status
inputFiles {
id
filename
filepath
filetype
fileSize
numPages
}
inputFile
inputFilename
resultFile
deleted
retrieved
errors
reviews {
id
createdAt
createdBy
completedAt
rejected
reviewType
notes
}
filepath
filename
filetype
submissionId
fileSize
numPages
}
inputFile
inputFilename
resultFile
outputFiles {
id
filepath
submissionId
componentId
createdAt
}
retrieved
autoReview {
id
submissionId
createdAt
createdBy
startedAt
completedAt
rejected
reviewType
notes
}
retries {
id
submissionId
previousErrors
previousStatus
retryErrors
}
reviews {
id
submissionId
createdAt
createdBy
startedAt
completedAt
rejected
reviewType
notes
}
reviewInProgress
}
pageInfo {
endCursor
hasNextPage
startCursor
endCursor
hasNextPage
aggregateCount
}
}
}
Expand Down
1 change: 1 addition & 0 deletions indico/types/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from .jobs import *
from .model_group import *
from .model import *
from .output_file import *
from .submission_file import *
from .submission import *
from .workflow import *
Expand Down
2 changes: 1 addition & 1 deletion indico/types/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __init__(self, **kwargs):
k = cc_to_snake(k)
if k in attrs:
attr_type = attrs[k]
if inspect.isclass(attr_type) and issubclass(attr_type, BaseType):
if v is not None and inspect.isclass(attr_type) and issubclass(attr_type, BaseType):
v = attrs[k](**v)

if attr_type == JSONType:
Expand Down
Loading

0 comments on commit 1547bd6

Please sign in to comment.