diff --git a/docs/source/upcoming_release_notes/79-augment_search_methods.rst b/docs/source/upcoming_release_notes/79-augment_search_methods.rst new file mode 100644 index 0000000..2792b2d --- /dev/null +++ b/docs/source/upcoming_release_notes/79-augment_search_methods.rst @@ -0,0 +1,24 @@ +79 augment search methods +################# + +API Breaks +---------- +- Client.search takes SearchTerms as *args rather than key-value pairs as **kwargs + +Features +-------- +- regex search on Entry text fields +- filter Entrys by tag +- filter Entrys by attribute value + +Bugfixes +-------- +- N/A + +Maintenance +----------- +- N/A + +Contributors +------------ +- shilorigins diff --git a/superscore/backends/core.py b/superscore/backends/core.py index a151312..9dac234 100644 --- a/superscore/backends/core.py +++ b/superscore/backends/core.py @@ -1,16 +1,28 @@ """ Base superscore data storage backend interface """ -from typing import Generator +from collections.abc import Container, Generator +from typing import NamedTuple, Union from uuid import UUID from superscore.model import Entry, Root +from superscore.type_hints import AnyEpicsType + +SearchTermValue = Union[AnyEpicsType, Container[AnyEpicsType], tuple[AnyEpicsType, ...]] +SearchTermType = tuple[str, str, SearchTermValue] + + +class SearchTerm(NamedTuple): + attr: str + operator: str + value: SearchTermValue class _Backend: """ Base class for data storage backend. """ + def get_entry(self, meta_id: UUID) -> Entry: """ Get entry with ``meta_id`` @@ -40,8 +52,18 @@ def update_entry(self, entry: Entry) -> None: """ raise NotImplementedError - def search(self, **search_kwargs) -> Generator[Entry, None, None]: - """Yield a Entry objects corresponding matching ``search_kwargs``""" + def search(self, *search_terms: SearchTermType) -> Generator[Entry, None, None]: + """ + Yield Entry objects matching all ``search_terms``. Each SearchTerm has the format + (, , ). Some operators take tuples as values. + + The supported operators are: + - eq (equals) + - lt (less than or equal to) + - gt (greater than or equal to) + - in + - like (fuzzy match, depends on type of value) + """ raise NotImplementedError @property diff --git a/superscore/backends/filestore.py b/superscore/backends/filestore.py index 7cf4684..61fa8b4 100644 --- a/superscore/backends/filestore.py +++ b/superscore/backends/filestore.py @@ -6,6 +6,7 @@ import json import logging import os +import re import shutil from dataclasses import fields, replace from typing import Any, Dict, Generator, Optional, Union @@ -13,9 +14,10 @@ from apischema import deserialize, serialize -from superscore.backends.core import _Backend +from superscore.backends.core import SearchTermType, SearchTermValue, _Backend from superscore.errors import BackendError from superscore.model import Entry, Root +from superscore.type_hints import AnyEpicsType from superscore.utils import build_abs_path logger = logging.getLogger(__name__) @@ -284,43 +286,63 @@ def delete_entry(self, entry: Entry) -> None: with self._load_and_store_context() as db: db.pop(entry.uuid, None) - def search(self, **search_kwargs) -> Generator[Entry, None, None]: + def search(self, *search_terms: SearchTermType) -> Generator[Entry, None, None]: """ - Search for an entry that matches ``search_kwargs``. - Keys are attributes on `Entry` subclasses - Values can be either a single value to match or a tuple of valid values - Currently does not support partial matches. + Return entries that match all ``search_terms``. + Keys are attributes on `Entry` subclasses, or special keywords. + Values can be a single value or a tuple of values depending on operator. """ with self._load_and_store_context() as db: for entry in db.values(): - match_found = True - for key, value in search_kwargs.items(): - # specific type handling, assuming is tuple - if key == "entry_type": - if not isinstance(entry, search_kwargs["entry_type"]): - match_found = False - - elif key == "start_time": - if value > entry.creation_time: - match_found = False - elif key == "end_time": - if entry.creation_time > value: - match_found = False - + conditions = [] + for attr, op, target in search_terms: # TODO: search for child pvs? - - # plain key-value match + if attr == "entry_type": + conditions.append(isinstance(entry, target)) else: - entry_value = getattr(entry, key, None) - if isinstance(value, tuple): - matched = entry_value in value - else: - matched = entry_value == value + try: + # check entry attribute by name + value = getattr(entry, attr) + conditions.append(self.compare(op, value, target)) + except AttributeError: + conditions.append(False) + if all(conditions): + yield entry - match_found = match_found and matched + @staticmethod + def compare(op: str, data: AnyEpicsType, target: SearchTermValue) -> bool: + """ + Return whether data and target satisfy the op comparator, typically durihg application + of a search filter. Possible values of op are detailed in _Backend.search - if match_found: - yield entry + Parameters + ---------- + op: str + one of the comparators that all backends must support, detailed in _Backend.search + data: AnyEpicsType | Tuple[AnyEpicsType] + data from an Entry that is being used to decide whether the Entry passes a filter + target: AnyEpicsType | Tuple[AnyEpicsType] + the filter value + + Returns + ------- + bool + whether data and target satisfy the op condition + """ + if op == "eq": + return data == target + elif op == "lt": + return data <= target + elif op == "gt": + return data >= target + elif op == "in": + return data in target + elif op == "like": + if isinstance(data, UUID): + data = str(data) + return re.search(target, data) + else: + raise ValueError(f"SearchTerm does not support operator \"{op}\"") @contextlib.contextmanager def _load_and_store_context(self) -> Generator[Dict[UUID, Any], None, None]: diff --git a/superscore/client.py b/superscore/client.py index a3265cb..2c46300 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -7,7 +7,7 @@ from uuid import UUID from superscore.backends import get_backend -from superscore.backends.core import _Backend +from superscore.backends.core import SearchTerm, SearchTermType, _Backend from superscore.control_layers import ControlLayer, EpicsData from superscore.control_layers.status import TaskStatus from superscore.errors import CommunicationError @@ -147,9 +147,26 @@ def find_config() -> Path: # If found nothing raise OSError("No superscore configuration file found. Check SUPERSCORE_CFG.") - def search(self, **post) -> Generator[Entry, None, None]: - """Search by key-value pair. Can search by any field, including id""" - return self.backend.search(**post) + def search(self, *post: SearchTermType) -> Generator[Entry, None, None]: + """ + Search backend for entries matching all SearchTerms in ``post``. Can search by any + field, plus some special keywords. Backends support operators listed in _Backend.search. + Some operators are supported in the UI / client and must be converted before being + passed to the backend. + """ + new_search_terms = [] + for search_term in post: + if not isinstance(search_term, SearchTerm): + search_term = SearchTerm(*search_term) + if search_term.operator == 'isclose': + target, rel_tol, abs_tol = search_term.value + lower = target - target * rel_tol - abs_tol + upper = target + target * rel_tol + abs_tol + new_search_terms.append(SearchTerm(search_term.attr, 'gt', lower)) + new_search_terms.append(SearchTerm(search_term.attr, 'lt', upper)) + else: + new_search_terms.append(search_term) + return self.backend.search(*new_search_terms) def save(self, entry: Entry): """Save information in ``entry`` to database""" diff --git a/superscore/tests/test_backend.py b/superscore/tests/test_backend.py index d62cd69..67016f4 100644 --- a/superscore/tests/test_backend.py +++ b/superscore/tests/test_backend.py @@ -1,8 +1,9 @@ +from enum import Flag, auto from uuid import UUID import pytest -from superscore.backends.core import _Backend +from superscore.backends.core import SearchTerm, _Backend from superscore.errors import (BackendError, EntryExistsError, EntryNotFoundError) from superscore.model import Collection, Parameter, Snapshot @@ -73,42 +74,118 @@ def test_delete_entry(backends: _Backend): def test_search_entry(backends: _Backend): # Given an entry we know is in the backend results = backends.search( - description='collection 1 defining some motor fields' + SearchTerm('description', 'eq', 'collection 1 defining some motor fields') ) assert len(list(results)) == 1 # Search by field name results = backends.search( - uuid=UUID('ffd668d3-57d9-404e-8366-0778af7aee61') + SearchTerm('uuid', 'eq', UUID('ffd668d3-57d9-404e-8366-0778af7aee61')) ) assert len(list(results)) == 1 # Search by field name - results = backends.search(data=2) + results = backends.search( + SearchTerm('data', 'eq', 2) + ) assert len(list(results)) == 3 # Search by field name results = backends.search( - uuid=UUID('ecb42cdb-b703-4562-86e1-45bd67a2ab1a'), data=2 + SearchTerm('uuid', 'eq', UUID('ecb42cdb-b703-4562-86e1-45bd67a2ab1a')), + SearchTerm('data', 'eq', 2) ) assert len(list(results)) == 1 - results = backends.search(entry_type=Snapshot,) + results = backends.search( + SearchTerm('entry_type', 'eq', Snapshot) + ) assert len(list(results)) == 1 - results = backends.search(entry_type=(Snapshot, Collection)) + results = backends.search( + SearchTerm('entry_type', 'in', (Snapshot, Collection)) + ) assert len(list(results)) == 2 + results = backends.search( + SearchTerm('data', 'lt', 3) + ) + assert len(list(results)) == 3 + + results = backends.search( + SearchTerm('data', 'gt', 3) + ) + assert len(list(results)) == 1 + + +@pytest.mark.parametrize('backends', [0], indirect=True) +def test_fuzzy_search(backends: _Backend): + results = list(backends.search( + SearchTerm('description', 'like', 'motor')) + ) + assert len(results) == 4 + + results = list(backends.search( + SearchTerm('description', 'like', 'motor field (?!PREC)')) + ) + assert len(results) == 2 + + results = list(backends.search( + SearchTerm('uuid', 'like', '17cc6ebf')) + ) + assert len(results) == 1 + + +@pytest.mark.parametrize('backends', [0], indirect=True) +def test_tag_search(backends: _Backend): + results = list(backends.search( + SearchTerm('tags', 'gt', set()) + )) + assert len(results) == 2 # only the Collection and Snapshot have .tags + + class Tag(Flag): + T1 = auto() + T2 = auto() + + results[0].tags = {Tag.T1} + results[1].tags = {Tag.T1, Tag.T2} + backends.update_entry(results[0]) + backends.update_entry(results[1]) + + results = list(backends.search( + SearchTerm('tags', 'gt', {Tag.T1}) + )) + assert len(results) == 2 + + results = list(backends.search( + SearchTerm('tags', 'gt', {Tag.T1, Tag.T2}) + )) + assert len(results) == 1 + + +@pytest.mark.parametrize('backends', [0], indirect=True) +def test_search_error(backends: _Backend): + with pytest.raises(TypeError): + results = backends.search( + SearchTerm('data', 'like', 5) + ) + list(results) + with pytest.raises(ValueError): + results = backends.search( + SearchTerm('data', 'near', 5) + ) + list(results) + @pytest.mark.parametrize('backends', [0], indirect=True) def test_update_entry(backends: _Backend): # grab an entry from the database and modify it. entry = list(backends.search( - description='collection 1 defining some motor fields' + SearchTerm('description', 'eq', 'collection 1 defining some motor fields') ))[0] old_uuid = entry.uuid entry.description = 'new_description' backends.update_entry(entry) new_entry = list(backends.search( - description='new_description' + SearchTerm('description', 'eq', 'new_description') ))[0] new_uuid = new_entry.uuid diff --git a/superscore/tests/test_client.py b/superscore/tests/test_client.py index 405a25c..a5659e3 100644 --- a/superscore/tests/test_client.py +++ b/superscore/tests/test_client.py @@ -4,6 +4,7 @@ import pytest +from superscore.backends.core import SearchTerm from superscore.backends.filestore import FilestoreBackend from superscore.client import Client from superscore.control_layers import EpicsData @@ -136,3 +137,15 @@ def test_find_config(sscore_cfg: str): # explicit SUPERSCORE_CFG env var supercedes XDG_CONFIG_HOME os.environ['SUPERSCORE_CFG'] = 'other/cfg' assert 'other/cfg' == Client.find_config() + + +def test_search(sample_client): + results = list(sample_client.search( + ('data', 'isclose', (4, 0, 0)) + )) + assert len(results) == 0 + + results = list(sample_client.search( + SearchTerm(operator='isclose', attr='data', value=(4, .5, 1)) + )) + assert len(results) == 4 diff --git a/superscore/widgets/page/collection_builder.py b/superscore/widgets/page/collection_builder.py index f06f4bb..912d7d0 100644 --- a/superscore/widgets/page/collection_builder.py +++ b/superscore/widgets/page/collection_builder.py @@ -229,8 +229,8 @@ def add_pv(self): def update_collection_choices(self): """update collection choices based on line edit""" - search_kwargs = {'entry_type': (Collection,)} - self._coll_options = [res for res in self.client.search(**search_kwargs) + search_term = ('entry_type', 'eq', Collection) + self._coll_options = [res for res in self.client.search(search_term) if res not in (self.data.children, self)] logger.debug(f"Gathered {len(self._coll_options)} collections") self.coll_combo_box.clear() diff --git a/superscore/widgets/page/search.py b/superscore/widgets/page/search.py index 77633a3..f9a4795 100644 --- a/superscore/widgets/page/search.py +++ b/superscore/widgets/page/search.py @@ -7,6 +7,7 @@ from dateutil import tz from qtpy import QtCore, QtWidgets +from superscore.backends.core import SearchTerm from superscore.client import Client from superscore.model import Collection, Entry, Readback, Setpoint, Snapshot from superscore.widgets import ICON_MAP @@ -95,7 +96,7 @@ def setup_ui(self) -> None: self.name_subfilter_line_edit.textChanged.connect(self.subfilter_results) def _gather_search_terms(self) -> Dict[str, Any]: - search_kwargs = {} + search_terms = [] # type entry_type_list = [Snapshot, Collection, Setpoint, Readback] @@ -106,39 +107,43 @@ def _gather_search_terms(self) -> Dict[str, Any]: if not checkbox.isChecked(): entry_type_list.remove(entry_type) - search_kwargs["entry_type"] = tuple(entry_type_list) + search_terms.append(SearchTerm('entry_type', 'eq', tuple(entry_type_list))) # name name = self.name_line_edit.text() if name: - search_kwargs['title'] = tuple(n.strip() for n in name.split(',')) + search_terms.append( + SearchTerm('title', 'in', tuple(n.strip() for n in name.split(','))) + ) # description desc = self.desc_line_edit.text() if desc: - search_kwargs['description'] = desc + search_terms.append(SearchTerm('description', 'like', desc)) # TODO: sort out PVs pvs = self.pv_line_edit.text() if pvs: - search_kwargs['pvs'] = tuple(pv.strip() for pv in pvs.split(',')) + search_terms.append( + SearchTerm('pvs', 'in', tuple(pv.strip() for pv in pvs.split(','))) + ) # time start_dt = self.start_dt_edit.dateTime().toPyDateTime() - search_kwargs['start_time'] = start_dt.astimezone(tz.UTC) + search_terms.append(SearchTerm('creation_time', 'gt', start_dt.astimezone(tz.UTC))) end_dt = self.end_dt_edit.dateTime().toPyDateTime() - search_kwargs['end_time'] = end_dt.astimezone(tz.UTC) + search_terms.append(SearchTerm('creation_time', 'lt', end_dt.astimezone(tz.UTC))) - logger.debug(f'gathered search terms: {search_kwargs}') - return search_kwargs + logger.debug(f'gathered search terms: {search_terms}') + return search_terms def show_current_filter(self) -> None: """ Gather filter options and update source model with valid entries """ # gather filter details - search_kwargs = self._gather_search_terms() - entries = self.client.search(**search_kwargs) + search_terms = self._gather_search_terms() + entries = self.client.search(*search_terms) # update source table model self.model.modelAboutToBeReset.emit() diff --git a/superscore/widgets/views.py b/superscore/widgets/views.py index c827690..76cbca2 100644 --- a/superscore/widgets/views.py +++ b/superscore/widgets/views.py @@ -16,6 +16,7 @@ import qtawesome as qta from qtpy import QtCore, QtGui, QtWidgets +from superscore.backends.core import SearchTerm from superscore.client import Client from superscore.control_layers import EpicsData from superscore.model import (Collection, Entry, Nestable, Parameter, Readback, @@ -66,7 +67,7 @@ def __init__( def fill_uuids(self, client: Optional[Client] = None) -> None: """Fill this item's data if it is a uuid, using ``client``""" if isinstance(self._data, UUID) and client is not None: - self._data = list(client.search(uuid=self._data))[0] + self._data = list(client.search(SearchTerm('uuid', 'eq', self._data)))[0] def data(self, column: int) -> Any: """