-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loading status checks…
#1 - use server-side processing for runs table, update deps, random m…
…ypy/flake8 fixes Signed-off-by: Lance Drane <[email protected]>
1 parent
4b689e3
commit 607b2ac
Showing
8 changed files
with
506 additions
and
145 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,224 @@ | ||
from __future__ import annotations | ||
|
||
from copy import deepcopy | ||
from typing import Any, Callable, Literal | ||
|
||
|
||
class SortParamException(Exception): | ||
def __init__(self, property: str, message: str, *args: object) -> None: | ||
super().__init__(*args) | ||
self.property = property | ||
self.message = message | ||
|
||
|
||
def _parse_sort_arguments( | ||
request: dict[str, Any], allowed_props: list[str] | ||
) -> dict[str, int]: | ||
"""Parse sort arguments from DataTables | ||
Params: | ||
request: DataTables server-side processing request argument | ||
allowed_props: list of allowed properties. | ||
If the request tries to get a property not in this list, raise SortParamException | ||
Returns: | ||
dictionary of property to sort direction. | ||
The dictionary should have the property names as keys, | ||
and the values should either be "1" (sort ASC) or "-1" (sort DESC). | ||
Raises: | ||
SortParamException - if any params are not properly formatted. Should never be raised if | ||
DataTables queries this API, but may be raised if the API is used directly elsewhere. | ||
""" | ||
sort_dict: dict[str, int] = {} | ||
columns: list[Any] = request.get("columns", None) | ||
if not isinstance(columns, list): | ||
raise SortParamException("columns", "must be an array") | ||
for idx, sort_args in enumerate(request.get("order", [])): | ||
col_idx = 0 | ||
err = False | ||
try: | ||
col_idx = sort_args["column"] | ||
if col_idx >= len(allowed_props): | ||
err = True | ||
except KeyError: | ||
err = True | ||
if err: | ||
raise SortParamException(f"order[{idx}]", "invalid column reference") | ||
try: | ||
column = columns[col_idx] | ||
except IndexError: | ||
raise SortParamException( | ||
f'order[{idx}]["column"]', "invalid column reference" | ||
) | ||
|
||
try: | ||
sort_dir = sort_args["dir"] | ||
if sort_dir == "asc": | ||
sort_code = 1 | ||
elif sort_dir == "desc": | ||
sort_code = -1 | ||
else: | ||
raise SortParamException( | ||
f'order[{idx}]["dir"]', 'must be either "asc" or "desc"' | ||
) | ||
except KeyError: | ||
raise SortParamException( | ||
f'order[{idx}]["dir"]', 'must be either "asc" or "desc"' | ||
) | ||
|
||
if not isinstance(column, dict): | ||
raise SortParamException( | ||
f"columns[{col_idx}]", "must be a valid DataTables column object" | ||
) | ||
|
||
if column.get("orderable", False): | ||
err = False | ||
try: | ||
dataname = column["data"] | ||
# if "columns" were defined client-side | ||
if isinstance(dataname, str): | ||
if dataname in allowed_props: | ||
sort_dict[dataname] = sort_code | ||
else: | ||
raise SortParamException( | ||
f"columns[{col_idx}][data]", "must be a valid property name" | ||
) | ||
# if "columns" were not defined client-side | ||
elif isinstance(dataname, int): | ||
try: | ||
sort_dict[allowed_props[dataname]] = sort_code | ||
except IndexError: | ||
raise SortParamException( | ||
f"columns[{col_idx}][data]", | ||
"must be a valid property index", | ||
) | ||
else: | ||
raise SortParamException( | ||
f"columns[{col_idx}][data]", | ||
"must be a valid property name or property index", | ||
) | ||
except KeyError: | ||
raise SortParamException( | ||
f"columns[{col_idx}]", 'missing "data" property' | ||
) | ||
|
||
if sort_dict: | ||
# enforce sort consistency in case all other properties are equal | ||
sort_dict["_id"] = 1 | ||
|
||
return sort_dict | ||
|
||
|
||
def _add_search_terms( | ||
search_terms: list[str], allowed_props: list[str] | ||
) -> dict[str, list[dict[str, Any]]]: | ||
"""Parse global search option from DataTables | ||
This simply adds an ignore-case regex search option for every column. | ||
Params: | ||
- search_terms: non-empty list of strings | ||
- allowed_props: specific columns we want to search by | ||
Returns: | ||
value to update the $match filter with | ||
""" | ||
return { | ||
"$and": [ | ||
{ | ||
"$or": [ | ||
{column: {"$regex": search_term, "$options": "i"}} | ||
for column in allowed_props | ||
] | ||
} | ||
for search_term in search_terms | ||
] | ||
} | ||
|
||
|
||
# potentially useful reference: https://github.com/pjosols/mongo-datatables | ||
# (NOTE: this library does NOT attempt to validate the input, so don't use it directly) | ||
def get_datatables_results( | ||
request: dict[str, Any], | ||
allowed_props: list[str], | ||
data_query_fn: Callable[[dict[str, Any], int, int, dict[str, int]], list[Any]], | ||
count_query_fn: Callable[[dict[str, Any]], int], | ||
base_filter: dict[str, Any] | None = None, | ||
) -> tuple[Literal[False], list[tuple[str, str]]] | tuple[Literal[True], dict[str, Any]]: | ||
""" | ||
This function is intended to be a generic mechanism when interacting with a DataTables request. | ||
The DataTables API specification is available at: https://datatables.net/manual/server-side | ||
Params: | ||
request: the parameters that DataTables provides. | ||
Note that it is assumed that this has already been parsed into a dictionary, | ||
this makes no assumptions about what web framework you're using. | ||
HOWEVER - the function will perform validation on the request object from here. | ||
(TODO - maybe use Pydantic or Msgspec to validate the data structure?) | ||
allowed_props: list of permitted properties to query for. DataTables sends in arbitrary | ||
values for its columns, but we need to enforce this on the server-side and never assume | ||
that client-side data is valid. | ||
data_query_fn: Callback function which takes in a filter, skip amount, length amount, | ||
and sort orders, and returns the actual data from Mongo. | ||
count_query_fn: Callback function which takes in a filter, | ||
and returns the number of documents from the query | ||
base_filter: Optional parameter, this is a starting $match MongoDB query | ||
that can be appended to if the frontend sends search instructions. | ||
(note that the parameter will not be modified, we instead make a copy of it internally) | ||
Returns: | ||
always a two-tuple value: | ||
- OnError: False, followed by a list of tuples. The first entry is the property path string, | ||
the second entry is the error string. | ||
- OnSuccess: True, followed by the DataTables response (as a dictionary - not serialized yet) | ||
""" | ||
if not isinstance(request, dict): | ||
return False, ( | ||
("<BASE>", "query parameter must be a DataTables JSON object string") | ||
) | ||
|
||
errors: list[tuple[str, str]] = [] | ||
|
||
draw = request.get("draw", 0) | ||
if not isinstance(draw, int) or draw < 0: | ||
errors.append(("draw", "must be a non-negative integer")) | ||
|
||
start = request.get("start", 0) | ||
if not isinstance(start, int) or start < 0: | ||
errors.append(("start", "must be a non-negative integer")) | ||
|
||
length = request.get("length", 20) | ||
if not isinstance(length, int) or length < 1: | ||
errors.append(("length", "must be a positive integer")) | ||
|
||
try: | ||
sort_args = _parse_sort_arguments(request, allowed_props) | ||
except SortParamException as e: | ||
sort_args = {} | ||
errors.append((e.property, e.message)) | ||
|
||
if errors: | ||
return False, errors | ||
|
||
# TODO: currently only checking global search, but DataTables API allows for per-column search | ||
where_filter = deepcopy(base_filter) if base_filter else {} | ||
search = request.get("search") | ||
if isinstance(search, dict): | ||
search_value = search.get("value") | ||
if isinstance(search_value, str): | ||
# TODO should probably sanitize this search result | ||
search_values = search_value.split() | ||
if search_values: | ||
where_filter.update(_add_search_terms(search_values, allowed_props)) | ||
|
||
return ( | ||
True, | ||
{ | ||
"draw": draw, | ||
"recordsTotal": count_query_fn(base_filter or {}), | ||
"recordsFiltered": count_query_fn(where_filter), | ||
"data": data_query_fn(where_filter, start, length, sort_args), | ||
}, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,77 +1,68 @@ | ||
$(document).ready( function () { | ||
$('#runs-table').DataTable( { | ||
ajax: { | ||
url: '/api/runs', | ||
dataSrc: '' | ||
}, | ||
deferRender: true, | ||
responsive: true, | ||
order: [[ 0, 'desc' ]], | ||
columns: [ | ||
{data: 'runid', | ||
render: function(data, type, row, meta){ | ||
if(type === 'display'){ | ||
data = `<a href="/${data}">${data}</a>`; | ||
} | ||
return data;}, | ||
responsivePriority: 1}, | ||
{data: 'state', | ||
defaultContent: '', | ||
responsivePriority: 8}, | ||
{data: 'rcomment', | ||
defaultContent: '', | ||
responsivePriority: 4}, | ||
{data: 'simname', | ||
defaultContent: '', | ||
responsivePriority: 2}, | ||
{data: 'host', | ||
defaultContent: ''}, | ||
{data: 'user', | ||
defaultContent: '', | ||
responsivePriority: 5}, | ||
{data: 'startat', | ||
defaultContent: '', | ||
responsivePriority: 6}, | ||
{data: 'stopat', | ||
defaultContent: '', | ||
responsivePriority: 7}, | ||
{data: 'walltime', | ||
defaultContent: '', | ||
responsivePriority: 3} | ||
], | ||
buttons: { | ||
buttons: [ | ||
{ | ||
text: 'Reload', | ||
className: 'btn btn-primary', | ||
action: function ( e, dt, node, config ) { | ||
dt.ajax.reload(); | ||
} | ||
} | ||
], | ||
dom: { | ||
button: { | ||
className: 'btn' | ||
} | ||
} | ||
}, | ||
processing: true, | ||
dom: | ||
"<'row'<'col-sm-12 col-md-6'B><'col-sm-12 col-md-6'f>>" + | ||
"<'row'<'col-sm-12'tr>>" + | ||
"<'row'<'col-sm-12 col-md-5'l><'col-sm-12 col-md-3'i><'col-sm-12 col-md-4'p>>", | ||
createdRow: function(row, data, dataIndex){ | ||
if( data['state'] == 'Completed'){ | ||
if (data['ok']) { | ||
$(row).addClass('table-success'); | ||
} else { | ||
$(row).addClass('table-danger'); | ||
} | ||
} | ||
else if (data['state'] == 'Timeout') { | ||
$(row).addClass('table-warning'); | ||
} | ||
$(document).ready(function () { | ||
$("#runs-table").DataTable({ | ||
ajax: { | ||
url: "/api/runs-datatables", | ||
// require query parameter for "data" to be JSON-encoded string | ||
data: (d) => ({ data: JSON.stringify(d) }), | ||
}, | ||
serverSide: true, | ||
processing: true, | ||
// uncomment this if we find that search takes too long | ||
// search: { | ||
// return: true, | ||
// }, | ||
responsive: true, | ||
order: [[0, "desc"]], | ||
columns: [ | ||
{ | ||
data: "runid", | ||
render: function (data, type, row, meta) { | ||
if (type === "display") { | ||
data = `<a href="/${data}">${data}</a>`; | ||
} | ||
return data; | ||
}, | ||
responsivePriority: 1, | ||
}, | ||
{ data: "state", defaultContent: "", responsivePriority: 8 }, | ||
{ data: "rcomment", defaultContent: "", responsivePriority: 4 }, | ||
{ data: "simname", defaultContent: "", responsivePriority: 2 }, | ||
{ data: "host", defaultContent: "" }, | ||
{ data: "user", defaultContent: "", responsivePriority: 5 }, | ||
{ data: "startat", defaultContent: "", responsivePriority: 6 }, | ||
{ data: "stopat", defaultContent: "", responsivePriority: 7 }, | ||
{ data: "walltime", defaultContent: "", responsivePriority: 3 }, | ||
], | ||
buttons: { | ||
buttons: [ | ||
{ | ||
text: "Reload", | ||
className: "btn btn-primary", | ||
action: function (e, dt, node, config) { | ||
dt.ajax.reload(); | ||
}, | ||
}, | ||
], | ||
dom: { | ||
button: { | ||
className: "btn", | ||
}, | ||
}, | ||
}, | ||
dom: | ||
"<'row'<'col-sm-12 col-md-6'B><'col-sm-12 col-md-6'f>>" + | ||
"<'row'<'col-sm-12'tr>>" + | ||
"<'row'<'col-sm-12 col-md-5'l><'col-sm-12 col-md-3'i><'col-sm-12 col-md-4'p>>", | ||
createdRow: function (row, data, dataIndex) { | ||
if (data["state"] == "Completed") { | ||
if (data["ok"]) { | ||
$(row).addClass("table-success"); | ||
} else { | ||
$(row).addClass("table-danger"); | ||
} | ||
} | ||
); | ||
} ); | ||
} else if (data["state"] == "Timeout") { | ||
$(row).addClass("table-warning"); | ||
} | ||
}, | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,39 +1,44 @@ | ||
<!doctype html> | ||
<!DOCTYPE html> | ||
<html lang="en"> | ||
<head> | ||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> | ||
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.1.3/css/bootstrap.min.css"/> | ||
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs5/jq-3.6.0/dt-1.12.1/b-2.2.3/r-2.3.0/datatables.min.css"/> | ||
<meta | ||
name="viewport" | ||
content="width=device-width, initial-scale=1, shrink-to-fit=no" | ||
/> | ||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/css/bootstrap.min.css" integrity="sha384-9ndCyUaIbzAi2FUVXJi0CjmCapSmO7SnpJef0486qhLnuZ2cdeRhO02iuK6FUUVM" crossorigin="anonymous"> | ||
<link rel="stylesheet" href="https://cdn.datatables.net/v/bs5/jq-3.7.0/dt-2.0.5/b-3.0.2/r-3.0.2/datatables.min.css" integrity="sha384-weUIDgnB1MxpCon++brXK2lxeXAwjWqmkpbur3C8scS+ghoeb+cCjYG1T8RrZX3W" crossorigin="anonymous"> | ||
|
||
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.1.3/js/bootstrap.bundle.min.js" integrity="sha256-9SEPo+fwJFpMUet/KACSwO+Z/dKMReF9q4zFhU/fT9M=" crossorigin="anonymous"></script> | ||
<script type="text/javascript" src="https://cdn.datatables.net/v/bs5/jq-3.6.0/dt-1.12.1/b-2.2.3/r-2.3.0/datatables.min.js" integrity="sha256-vlTf/vLdvJS3Kn9quSfyTf+zGawqN0jCuN6ziTEbWso=" crossorigin="anonymous"></script> | ||
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/js/bootstrap.bundle.min.js" integrity="sha384-geWF76RCwLtnZ8qwWowPQNguL3RmwHVBC9FhGdlKrxdiJJigb/j/68SIy3Te4Bkz" crossorigin="anonymous"></script> | ||
<script src="https://cdn.datatables.net/v/bs5/jq-3.7.0/dt-2.0.5/b-3.0.2/r-3.0.2/datatables.min.js" integrity="sha384-3rjNyGE1FfJ85xdeik+xKUXOqr/7SvLVeHofViooMbhywDuChe3dioclA9/wsEn1" crossorigin="anonymous"></script> | ||
<title>IPS Portal - {% block title %}{% endblock %}</title> | ||
{% block head %}{% endblock %} | ||
</head> | ||
|
||
<body> | ||
|
||
<nav class="navbar navbar-expand navbar-light bg-light mb-4"> | ||
<div class="container-fluid"> | ||
<div class="navbar-brand">IPS Portal</div> | ||
<div class="navbar" id="navbarNav"> | ||
<ul class="navbar-nav"> | ||
<li class="nav-item"> | ||
<a class="nav-link" href="/">Home</a> | ||
</li> | ||
<li class="nav-item"> | ||
<a class="nav-link" href="https://ips-framework.readthedocs.io/en/latest/user_guides/portal_guides.html">Documentation</a> | ||
</li> | ||
</ul> | ||
</div> | ||
<div class="navbar" id="navbarNav"> | ||
<ul class="navbar-nav"> | ||
<li class="nav-item"> | ||
<a class="nav-link" href="/">Home</a> | ||
</li> | ||
<li class="nav-item"> | ||
<a | ||
class="nav-link" | ||
href="https://ips-framework.readthedocs.io/en/latest/user_guides/portal_guides.html" | ||
>Documentation</a | ||
> | ||
</li> | ||
</ul> | ||
</div> | ||
</div> | ||
</nav> | ||
|
||
<main> | ||
<section style="max-width:1600px; width: 90%; margin: 0 auto;"> | ||
{% block content %}{% endblock %} | ||
</section> | ||
<section style="max-width: 1600px; width: 90%; margin: 0 auto"> | ||
{% block content %}{% endblock %} | ||
</section> | ||
</main> | ||
|
||
</body> | ||
</html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
from __future__ import annotations | ||
|
||
ALLOWED_PROPS_RUN = [ | ||
'runid', | ||
'state', | ||
'rcomment', | ||
'simname', | ||
'host', | ||
'user', | ||
'startat', | ||
'stopat', | ||
'walltime', | ||
] | ||
""" | ||
Properties we allow sort queries on for the runs table. | ||
THIS MUST MATCH THE CLIENT SIDE CONFIGURATION. | ||
""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters