From 4d1a9fa0474d26da8e6445e582c7241b985968d5 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Thu, 22 Aug 2024 10:20:24 -0700 Subject: [PATCH 01/11] MNT: refactor FilestoreBackend.search --- superscore/backends/filestore.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/superscore/backends/filestore.py b/superscore/backends/filestore.py index 7cf4684..2e619e7 100644 --- a/superscore/backends/filestore.py +++ b/superscore/backends/filestore.py @@ -293,33 +293,22 @@ def search(self, **search_kwargs) -> Generator[Entry, None, None]: """ with self._load_and_store_context() as db: for entry in db.values(): - match_found = True + conditions = [] for key, value in search_kwargs.items(): - # specific type handling, assuming is tuple - if key == "entry_type": - if not isinstance(entry, search_kwargs["entry_type"]): - match_found = False - + if key == "entry_type": # specific type handling, assume value is tuple + conditions.append(isinstance(entry, value)) elif key == "start_time": - if value > entry.creation_time: - match_found = False + conditions.append(entry.creation_time > value) elif key == "end_time": - if entry.creation_time > value: - match_found = False - + conditions.append(entry.creation_time < value) # TODO: search for child pvs? - - # plain key-value match - else: + else: # plain key-value match entry_value = getattr(entry, key, None) if isinstance(value, tuple): - matched = entry_value in value + conditions.append(entry_value in value) else: - matched = entry_value == value - - match_found = match_found and matched - - if match_found: + conditions.append(entry_value == value) + if all(conditions): yield entry @contextlib.contextmanager From 847e8b8e063a603fa2e7bc7a959c75fec0be842b Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Thu, 22 Aug 2024 10:30:24 -0700 Subject: [PATCH 02/11] ENH: implement and integrate SearchTerm SearchTerm is a namedtuple with format (, , ). should be an attribute of Entry, or one of the specially handled keywords. is a string telling the backend which function to use to filter . is the target value of ; can be a single value or a tuple depending on . --- superscore/backends/core.py | 18 +++++++- superscore/backends/filestore.py | 69 +++++++++++++++++++++++-------- superscore/client.py | 9 ++-- superscore/tests/test_backend.py | 48 +++++++++++++++++---- superscore/widgets/page/search.py | 27 +++++++----- superscore/widgets/views.py | 3 +- 6 files changed, 130 insertions(+), 44 deletions(-) diff --git a/superscore/backends/core.py b/superscore/backends/core.py index a151312..9b87b0a 100644 --- a/superscore/backends/core.py +++ b/superscore/backends/core.py @@ -1,16 +1,20 @@ """ Base superscore data storage backend interface """ +from collections import namedtuple from typing import Generator from uuid import UUID from superscore.model import Entry, Root +SearchTerm = namedtuple('SearchTerm', ('attr', 'operator', 'value')) + class _Backend: """ Base class for data storage backend. """ + def get_entry(self, meta_id: UUID) -> Entry: """ Get entry with ``meta_id`` @@ -40,8 +44,18 @@ def update_entry(self, entry: Entry) -> None: """ raise NotImplementedError - def search(self, **search_kwargs) -> Generator[Entry, None, None]: - """Yield a Entry objects corresponding matching ``search_kwargs``""" + def search(self, *search_terms) -> Generator[Entry, None, None]: + """ + Yield Entry objects matching all ``search_terms``. Each SearchTerm has the format + (, , ). Some operators take tuples as values. + + The supported operators are: + - eq (equals) + - lt (less than or equal to) + - gt (greater than or equal to) + - in + - like (fuzzy match, depends on type of value) + """ raise NotImplementedError @property diff --git a/superscore/backends/filestore.py b/superscore/backends/filestore.py index 2e619e7..72776d3 100644 --- a/superscore/backends/filestore.py +++ b/superscore/backends/filestore.py @@ -6,6 +6,7 @@ import json import logging import os +import re import shutil from dataclasses import fields, replace from typing import Any, Dict, Generator, Optional, Union @@ -284,33 +285,65 @@ def delete_entry(self, entry: Entry) -> None: with self._load_and_store_context() as db: db.pop(entry.uuid, None) - def search(self, **search_kwargs) -> Generator[Entry, None, None]: + def search(self, *search_terms) -> Generator[Entry, None, None]: """ - Search for an entry that matches ``search_kwargs``. - Keys are attributes on `Entry` subclasses - Values can be either a single value to match or a tuple of valid values - Currently does not support partial matches. + Return entries that match all ``search_terms``. + Keys are attributes on `Entry` subclasses, or special keywords. + Values can be a single value or a tuple of values depending on operator. """ with self._load_and_store_context() as db: for entry in db.values(): conditions = [] - for key, value in search_kwargs.items(): - if key == "entry_type": # specific type handling, assume value is tuple - conditions.append(isinstance(entry, value)) - elif key == "start_time": - conditions.append(entry.creation_time > value) - elif key == "end_time": - conditions.append(entry.creation_time < value) + for attr, op, target in search_terms: # TODO: search for child pvs? - else: # plain key-value match - entry_value = getattr(entry, key, None) - if isinstance(value, tuple): - conditions.append(entry_value in value) - else: - conditions.append(entry_value == value) + if attr == "entry_type": + conditions.append(isinstance(entry, target)) + else: + try: + # check entry attribute by name + value = getattr(entry, attr) + conditions.append(self.compare(op, value, target)) + except AttributeError: + conditions.append(False) if all(conditions): yield entry + @staticmethod + def compare(op: str, data, target) -> bool: + """ + Return whether data and target satisfy the op comparator, typically durihg application + of a search filter. Possible values of op are detailed in _Backend.search + + Parameters + ---------- + op: str + one of the comparators that all backends must support, detailed in _Backend.search + data: AnyEpicsType | Tuple[AnyEpicsType] + data from an Entry that is being used to decide whether the Entry passes a filter + target: AnyEpicsType | Tuple[AnyEpicsType] + the filter value + + Returns + ------- + bool + whether data and target satisfy the op condition + """ + if op == "eq": + return data == target + elif op == "lt": + return data <= target + elif op == "gt": + return data >= target + elif op == "in": + return data in target + elif op == "like": + if isinstance(data, str): + return re.search(target, data) + elif isinstance(data, (int, float)): + return data < 1.05 * target and data > .95 * target + else: + return NotImplemented + @contextlib.contextmanager def _load_and_store_context(self) -> Generator[Dict[UUID, Any], None, None]: """ diff --git a/superscore/client.py b/superscore/client.py index a3265cb..67db27c 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -147,9 +147,12 @@ def find_config() -> Path: # If found nothing raise OSError("No superscore configuration file found. Check SUPERSCORE_CFG.") - def search(self, **post) -> Generator[Entry, None, None]: - """Search by key-value pair. Can search by any field, including id""" - return self.backend.search(**post) + def search(self, *post) -> Generator[Entry, None, None]: + """ + Search backend for entries matching all SearchTerms in ``post``. Can search by any + field, plus some special keywords. Backends support operators listed in _Backend.search. + """ + return self.backend.search(*post) def save(self, entry: Entry): """Save information in ``entry`` to database""" diff --git a/superscore/tests/test_backend.py b/superscore/tests/test_backend.py index d62cd69..6b7fb07 100644 --- a/superscore/tests/test_backend.py +++ b/superscore/tests/test_backend.py @@ -2,7 +2,7 @@ import pytest -from superscore.backends.core import _Backend +from superscore.backends.core import SearchTerm, _Backend from superscore.errors import (BackendError, EntryExistsError, EntryNotFoundError) from superscore.model import Collection, Parameter, Snapshot @@ -73,42 +73,72 @@ def test_delete_entry(backends: _Backend): def test_search_entry(backends: _Backend): # Given an entry we know is in the backend results = backends.search( - description='collection 1 defining some motor fields' + SearchTerm('description', 'eq', 'collection 1 defining some motor fields') ) assert len(list(results)) == 1 # Search by field name results = backends.search( - uuid=UUID('ffd668d3-57d9-404e-8366-0778af7aee61') + SearchTerm('uuid', 'eq', UUID('ffd668d3-57d9-404e-8366-0778af7aee61')) ) assert len(list(results)) == 1 # Search by field name - results = backends.search(data=2) + results = backends.search( + SearchTerm('data', 'eq', 2) + ) assert len(list(results)) == 3 # Search by field name results = backends.search( - uuid=UUID('ecb42cdb-b703-4562-86e1-45bd67a2ab1a'), data=2 + SearchTerm('uuid', 'eq', UUID('ecb42cdb-b703-4562-86e1-45bd67a2ab1a')), + SearchTerm('data', 'eq', 2) ) assert len(list(results)) == 1 - results = backends.search(entry_type=Snapshot,) + results = backends.search( + SearchTerm('entry_type', 'eq', Snapshot) + ) assert len(list(results)) == 1 - results = backends.search(entry_type=(Snapshot, Collection)) + results = backends.search( + SearchTerm('entry_type', 'in', (Snapshot, Collection)) + ) assert len(list(results)) == 2 + results = backends.search( + SearchTerm('data', 'lt', 3) + ) + assert len(list(results)) == 3 + + results = backends.search( + SearchTerm('data', 'gt', 3) + ) + assert len(list(results)) == 1 + + +@pytest.mark.parametrize('backends', [0], indirect=True) +def test_fuzzy_search(backends: _Backend): + results = list(backends.search( + SearchTerm('description', 'like', 'motor')) + ) + assert len(results) == 4 + + results = list(backends.search( + SearchTerm('description', 'like', 'motor field (?!PREC)')) + ) + assert len(results) == 2 + @pytest.mark.parametrize('backends', [0], indirect=True) def test_update_entry(backends: _Backend): # grab an entry from the database and modify it. entry = list(backends.search( - description='collection 1 defining some motor fields' + SearchTerm('description', 'eq', 'collection 1 defining some motor fields') ))[0] old_uuid = entry.uuid entry.description = 'new_description' backends.update_entry(entry) new_entry = list(backends.search( - description='new_description' + SearchTerm('description', 'eq', 'new_description') ))[0] new_uuid = new_entry.uuid diff --git a/superscore/widgets/page/search.py b/superscore/widgets/page/search.py index 77633a3..f9a4795 100644 --- a/superscore/widgets/page/search.py +++ b/superscore/widgets/page/search.py @@ -7,6 +7,7 @@ from dateutil import tz from qtpy import QtCore, QtWidgets +from superscore.backends.core import SearchTerm from superscore.client import Client from superscore.model import Collection, Entry, Readback, Setpoint, Snapshot from superscore.widgets import ICON_MAP @@ -95,7 +96,7 @@ def setup_ui(self) -> None: self.name_subfilter_line_edit.textChanged.connect(self.subfilter_results) def _gather_search_terms(self) -> Dict[str, Any]: - search_kwargs = {} + search_terms = [] # type entry_type_list = [Snapshot, Collection, Setpoint, Readback] @@ -106,39 +107,43 @@ def _gather_search_terms(self) -> Dict[str, Any]: if not checkbox.isChecked(): entry_type_list.remove(entry_type) - search_kwargs["entry_type"] = tuple(entry_type_list) + search_terms.append(SearchTerm('entry_type', 'eq', tuple(entry_type_list))) # name name = self.name_line_edit.text() if name: - search_kwargs['title'] = tuple(n.strip() for n in name.split(',')) + search_terms.append( + SearchTerm('title', 'in', tuple(n.strip() for n in name.split(','))) + ) # description desc = self.desc_line_edit.text() if desc: - search_kwargs['description'] = desc + search_terms.append(SearchTerm('description', 'like', desc)) # TODO: sort out PVs pvs = self.pv_line_edit.text() if pvs: - search_kwargs['pvs'] = tuple(pv.strip() for pv in pvs.split(',')) + search_terms.append( + SearchTerm('pvs', 'in', tuple(pv.strip() for pv in pvs.split(','))) + ) # time start_dt = self.start_dt_edit.dateTime().toPyDateTime() - search_kwargs['start_time'] = start_dt.astimezone(tz.UTC) + search_terms.append(SearchTerm('creation_time', 'gt', start_dt.astimezone(tz.UTC))) end_dt = self.end_dt_edit.dateTime().toPyDateTime() - search_kwargs['end_time'] = end_dt.astimezone(tz.UTC) + search_terms.append(SearchTerm('creation_time', 'lt', end_dt.astimezone(tz.UTC))) - logger.debug(f'gathered search terms: {search_kwargs}') - return search_kwargs + logger.debug(f'gathered search terms: {search_terms}') + return search_terms def show_current_filter(self) -> None: """ Gather filter options and update source model with valid entries """ # gather filter details - search_kwargs = self._gather_search_terms() - entries = self.client.search(**search_kwargs) + search_terms = self._gather_search_terms() + entries = self.client.search(*search_terms) # update source table model self.model.modelAboutToBeReset.emit() diff --git a/superscore/widgets/views.py b/superscore/widgets/views.py index c827690..76cbca2 100644 --- a/superscore/widgets/views.py +++ b/superscore/widgets/views.py @@ -16,6 +16,7 @@ import qtawesome as qta from qtpy import QtCore, QtGui, QtWidgets +from superscore.backends.core import SearchTerm from superscore.client import Client from superscore.control_layers import EpicsData from superscore.model import (Collection, Entry, Nestable, Parameter, Readback, @@ -66,7 +67,7 @@ def __init__( def fill_uuids(self, client: Optional[Client] = None) -> None: """Fill this item's data if it is a uuid, using ``client``""" if isinstance(self._data, UUID) and client is not None: - self._data = list(client.search(uuid=self._data))[0] + self._data = list(client.search(SearchTerm('uuid', 'eq', self._data)))[0] def data(self, column: int) -> Any: """ From a84c0103670dafb8ff2bcb366e3537932e13e4b5 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Mon, 9 Sep 2024 15:50:45 -0700 Subject: [PATCH 03/11] ENH: add 'like_with_tols' search operator This operator is converted in Client.search, but is provided as a way for the UI to pass user-provided tolerances into the search function --- superscore/backends/filestore.py | 5 +---- superscore/client.py | 16 ++++++++++++++-- superscore/tests/test_client.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/superscore/backends/filestore.py b/superscore/backends/filestore.py index 72776d3..337e399 100644 --- a/superscore/backends/filestore.py +++ b/superscore/backends/filestore.py @@ -339,10 +339,7 @@ def compare(op: str, data, target) -> bool: elif op == "like": if isinstance(data, str): return re.search(target, data) - elif isinstance(data, (int, float)): - return data < 1.05 * target and data > .95 * target - else: - return NotImplemented + return NotImplemented @contextlib.contextmanager def _load_and_store_context(self) -> Generator[Dict[UUID, Any], None, None]: diff --git a/superscore/client.py b/superscore/client.py index 67db27c..c1dd398 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -7,7 +7,7 @@ from uuid import UUID from superscore.backends import get_backend -from superscore.backends.core import _Backend +from superscore.backends.core import SearchTerm, _Backend from superscore.control_layers import ControlLayer, EpicsData from superscore.control_layers.status import TaskStatus from superscore.errors import CommunicationError @@ -151,8 +151,20 @@ def search(self, *post) -> Generator[Entry, None, None]: """ Search backend for entries matching all SearchTerms in ``post``. Can search by any field, plus some special keywords. Backends support operators listed in _Backend.search. + Some operators are supported in the UI / client and must be converted before being + passed to the backend. """ - return self.backend.search(*post) + new_search_terms = [] + for search_term in post: + if search_term.operator == 'like_with_tols': + target, rel_tol, abs_tol = search_term.value + lower = target - target * rel_tol - abs_tol + upper = target + target * rel_tol + abs_tol + new_search_terms.append(SearchTerm(search_term.attr, 'gt', lower)) + new_search_terms.append(SearchTerm(search_term.attr, 'lt', upper)) + else: + new_search_terms.append(search_term) + return self.backend.search(*new_search_terms) def save(self, entry: Entry): """Save information in ``entry`` to database""" diff --git a/superscore/tests/test_client.py b/superscore/tests/test_client.py index 405a25c..455c6fb 100644 --- a/superscore/tests/test_client.py +++ b/superscore/tests/test_client.py @@ -4,6 +4,7 @@ import pytest +from superscore.backends.core import SearchTerm from superscore.backends.filestore import FilestoreBackend from superscore.client import Client from superscore.control_layers import EpicsData @@ -136,3 +137,15 @@ def test_find_config(sscore_cfg: str): # explicit SUPERSCORE_CFG env var supercedes XDG_CONFIG_HOME os.environ['SUPERSCORE_CFG'] = 'other/cfg' assert 'other/cfg' == Client.find_config() + + +def test_search(sample_client): + results = list(sample_client.search( + SearchTerm('data', 'like_with_tols', (4, 0, 0)) + )) + assert len(results) == 0 + + results = list(sample_client.search( + SearchTerm('data', 'like_with_tols', (4, .5, 1)) + )) + assert len(results) == 4 From fc07479fea77922c63af2ff8c0177937cebc1d7c Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Tue, 10 Sep 2024 09:14:59 -0700 Subject: [PATCH 04/11] TST: add working tag search test --- superscore/tests/test_backend.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/superscore/tests/test_backend.py b/superscore/tests/test_backend.py index 6b7fb07..01afe98 100644 --- a/superscore/tests/test_backend.py +++ b/superscore/tests/test_backend.py @@ -1,3 +1,4 @@ +from enum import Flag, auto from uuid import UUID import pytest @@ -127,6 +128,33 @@ def test_fuzzy_search(backends: _Backend): assert len(results) == 2 +@pytest.mark.parametrize('backends', [0], indirect=True) +def test_tag_search(backends: _Backend): + results = list(backends.search( + SearchTerm('tags', 'gt', set()) + )) + assert len(results) == 2 # only the Collection and Snapshot have .tags + + class Tag(Flag): + T1 = auto() + T2 = auto() + + results[0].tags = {Tag.T1} + results[1].tags = {Tag.T1, Tag.T2} + backends.update_entry(results[0]) + backends.update_entry(results[1]) + + results = list(backends.search( + SearchTerm('tags', 'gt', {Tag.T1}) + )) + assert len(results) == 2 + + results = list(backends.search( + SearchTerm('tags', 'gt', {Tag.T1, Tag.T2}) + )) + assert len(results) == 1 + + @pytest.mark.parametrize('backends', [0], indirect=True) def test_update_entry(backends: _Backend): # grab an entry from the database and modify it. From bded804fb6a1fd42788212ac52fd26299e4edd53 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Fri, 13 Sep 2024 09:35:30 -0700 Subject: [PATCH 05/11] DOC: pre-release notes --- .../79-augment_search_methods.rst | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/source/upcoming_release_notes/79-augment_search_methods.rst diff --git a/docs/source/upcoming_release_notes/79-augment_search_methods.rst b/docs/source/upcoming_release_notes/79-augment_search_methods.rst new file mode 100644 index 0000000..2792b2d --- /dev/null +++ b/docs/source/upcoming_release_notes/79-augment_search_methods.rst @@ -0,0 +1,24 @@ +79 augment search methods +################# + +API Breaks +---------- +- Client.search takes SearchTerms as *args rather than key-value pairs as **kwargs + +Features +-------- +- regex search on Entry text fields +- filter Entrys by tag +- filter Entrys by attribute value + +Bugfixes +-------- +- N/A + +Maintenance +----------- +- N/A + +Contributors +------------ +- shilorigins From 4424438c75f38ce87fddc0cb9b90154b5ba8acd2 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Tue, 17 Sep 2024 10:28:01 -0700 Subject: [PATCH 06/11] MNT: support calling Client.search with bare tuples --- superscore/client.py | 2 ++ superscore/tests/test_client.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/superscore/client.py b/superscore/client.py index c1dd398..c1f0588 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -156,6 +156,8 @@ def search(self, *post) -> Generator[Entry, None, None]: """ new_search_terms = [] for search_term in post: + if not isinstance(search_term, SearchTerm): + search_term = SearchTerm(*search_term) if search_term.operator == 'like_with_tols': target, rel_tol, abs_tol = search_term.value lower = target - target * rel_tol - abs_tol diff --git a/superscore/tests/test_client.py b/superscore/tests/test_client.py index 455c6fb..8bba73f 100644 --- a/superscore/tests/test_client.py +++ b/superscore/tests/test_client.py @@ -141,11 +141,11 @@ def test_find_config(sscore_cfg: str): def test_search(sample_client): results = list(sample_client.search( - SearchTerm('data', 'like_with_tols', (4, 0, 0)) + ('data', 'like_with_tols', (4, 0, 0)) )) assert len(results) == 0 results = list(sample_client.search( - SearchTerm('data', 'like_with_tols', (4, .5, 1)) + SearchTerm(operator='like_with_tols', attr='data', value=(4, .5, 1)) )) assert len(results) == 4 From c472661f2728deb0ae16f30365d609e50fb4cac5 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Tue, 17 Sep 2024 11:32:05 -0700 Subject: [PATCH 07/11] MNT: raise errors if SearchTerm receives unsupported op or arg type --- superscore/backends/filestore.py | 6 +++--- superscore/tests/test_backend.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/superscore/backends/filestore.py b/superscore/backends/filestore.py index 337e399..d957328 100644 --- a/superscore/backends/filestore.py +++ b/superscore/backends/filestore.py @@ -337,9 +337,9 @@ def compare(op: str, data, target) -> bool: elif op == "in": return data in target elif op == "like": - if isinstance(data, str): - return re.search(target, data) - return NotImplemented + return re.search(target, data) + else: + raise ValueError(f"SearchTerm does not support operator \"{op}\"") @contextlib.contextmanager def _load_and_store_context(self) -> Generator[Dict[UUID, Any], None, None]: diff --git a/superscore/tests/test_backend.py b/superscore/tests/test_backend.py index 01afe98..d9182da 100644 --- a/superscore/tests/test_backend.py +++ b/superscore/tests/test_backend.py @@ -155,6 +155,20 @@ class Tag(Flag): assert len(results) == 1 +@pytest.mark.parametrize('backends', [0], indirect=True) +def test_search_error(backends: _Backend): + with pytest.raises(TypeError): + results = backends.search( + SearchTerm('data', 'like', 5) + ) + list(results) + with pytest.raises(ValueError): + results = backends.search( + SearchTerm('data', 'near', 5) + ) + list(results) + + @pytest.mark.parametrize('backends', [0], indirect=True) def test_update_entry(backends: _Backend): # grab an entry from the database and modify it. From 55ee758a608d07f2970b3f67a2e3ed066d5033f4 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Tue, 17 Sep 2024 11:36:45 -0700 Subject: [PATCH 08/11] MNT: rename 'like_with_tols' to 'isclose' --- superscore/client.py | 2 +- superscore/tests/test_client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/superscore/client.py b/superscore/client.py index c1f0588..c01dd34 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -158,7 +158,7 @@ def search(self, *post) -> Generator[Entry, None, None]: for search_term in post: if not isinstance(search_term, SearchTerm): search_term = SearchTerm(*search_term) - if search_term.operator == 'like_with_tols': + if search_term.operator == 'isclose': target, rel_tol, abs_tol = search_term.value lower = target - target * rel_tol - abs_tol upper = target + target * rel_tol + abs_tol diff --git a/superscore/tests/test_client.py b/superscore/tests/test_client.py index 8bba73f..a5659e3 100644 --- a/superscore/tests/test_client.py +++ b/superscore/tests/test_client.py @@ -141,11 +141,11 @@ def test_find_config(sscore_cfg: str): def test_search(sample_client): results = list(sample_client.search( - ('data', 'like_with_tols', (4, 0, 0)) + ('data', 'isclose', (4, 0, 0)) )) assert len(results) == 0 results = list(sample_client.search( - SearchTerm(operator='like_with_tols', attr='data', value=(4, .5, 1)) + SearchTerm(operator='isclose', attr='data', value=(4, .5, 1)) )) assert len(results) == 4 From a276aaf97fe70a679d0bc42cb12f78b83cb4af50 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Tue, 17 Sep 2024 09:15:23 -0700 Subject: [PATCH 09/11] MNT: type hint SearchTerm and search method parameters --- superscore/backends/core.py | 16 ++++++++++++---- superscore/backends/filestore.py | 7 ++++--- superscore/client.py | 4 ++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/superscore/backends/core.py b/superscore/backends/core.py index 9b87b0a..9dac234 100644 --- a/superscore/backends/core.py +++ b/superscore/backends/core.py @@ -1,13 +1,21 @@ """ Base superscore data storage backend interface """ -from collections import namedtuple -from typing import Generator +from collections.abc import Container, Generator +from typing import NamedTuple, Union from uuid import UUID from superscore.model import Entry, Root +from superscore.type_hints import AnyEpicsType -SearchTerm = namedtuple('SearchTerm', ('attr', 'operator', 'value')) +SearchTermValue = Union[AnyEpicsType, Container[AnyEpicsType], tuple[AnyEpicsType, ...]] +SearchTermType = tuple[str, str, SearchTermValue] + + +class SearchTerm(NamedTuple): + attr: str + operator: str + value: SearchTermValue class _Backend: @@ -44,7 +52,7 @@ def update_entry(self, entry: Entry) -> None: """ raise NotImplementedError - def search(self, *search_terms) -> Generator[Entry, None, None]: + def search(self, *search_terms: SearchTermType) -> Generator[Entry, None, None]: """ Yield Entry objects matching all ``search_terms``. Each SearchTerm has the format (, , ). Some operators take tuples as values. diff --git a/superscore/backends/filestore.py b/superscore/backends/filestore.py index d957328..d814899 100644 --- a/superscore/backends/filestore.py +++ b/superscore/backends/filestore.py @@ -14,9 +14,10 @@ from apischema import deserialize, serialize -from superscore.backends.core import _Backend +from superscore.backends.core import SearchTermType, SearchTermValue, _Backend from superscore.errors import BackendError from superscore.model import Entry, Root +from superscore.type_hints import AnyEpicsType from superscore.utils import build_abs_path logger = logging.getLogger(__name__) @@ -285,7 +286,7 @@ def delete_entry(self, entry: Entry) -> None: with self._load_and_store_context() as db: db.pop(entry.uuid, None) - def search(self, *search_terms) -> Generator[Entry, None, None]: + def search(self, *search_terms: SearchTermType) -> Generator[Entry, None, None]: """ Return entries that match all ``search_terms``. Keys are attributes on `Entry` subclasses, or special keywords. @@ -309,7 +310,7 @@ def search(self, *search_terms) -> Generator[Entry, None, None]: yield entry @staticmethod - def compare(op: str, data, target) -> bool: + def compare(op: str, data: AnyEpicsType, target: SearchTermValue) -> bool: """ Return whether data and target satisfy the op comparator, typically durihg application of a search filter. Possible values of op are detailed in _Backend.search diff --git a/superscore/client.py b/superscore/client.py index c01dd34..2c46300 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -7,7 +7,7 @@ from uuid import UUID from superscore.backends import get_backend -from superscore.backends.core import SearchTerm, _Backend +from superscore.backends.core import SearchTerm, SearchTermType, _Backend from superscore.control_layers import ControlLayer, EpicsData from superscore.control_layers.status import TaskStatus from superscore.errors import CommunicationError @@ -147,7 +147,7 @@ def find_config() -> Path: # If found nothing raise OSError("No superscore configuration file found. Check SUPERSCORE_CFG.") - def search(self, *post) -> Generator[Entry, None, None]: + def search(self, *post: SearchTermType) -> Generator[Entry, None, None]: """ Search backend for entries matching all SearchTerms in ``post``. Can search by any field, plus some special keywords. Backends support operators listed in _Backend.search. From 4e749df5b1e5921d5029797f1362b3d06d029572 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Thu, 19 Sep 2024 09:06:07 -0700 Subject: [PATCH 10/11] MNT: support searching UUIDs with 'like' operator --- superscore/backends/filestore.py | 2 ++ superscore/tests/test_backend.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/superscore/backends/filestore.py b/superscore/backends/filestore.py index d814899..61fa8b4 100644 --- a/superscore/backends/filestore.py +++ b/superscore/backends/filestore.py @@ -338,6 +338,8 @@ def compare(op: str, data: AnyEpicsType, target: SearchTermValue) -> bool: elif op == "in": return data in target elif op == "like": + if isinstance(data, UUID): + data = str(data) return re.search(target, data) else: raise ValueError(f"SearchTerm does not support operator \"{op}\"") diff --git a/superscore/tests/test_backend.py b/superscore/tests/test_backend.py index d9182da..67016f4 100644 --- a/superscore/tests/test_backend.py +++ b/superscore/tests/test_backend.py @@ -127,6 +127,11 @@ def test_fuzzy_search(backends: _Backend): ) assert len(results) == 2 + results = list(backends.search( + SearchTerm('uuid', 'like', '17cc6ebf')) + ) + assert len(results) == 1 + @pytest.mark.parametrize('backends', [0], indirect=True) def test_tag_search(backends: _Backend): From f73ef720c538a674a231ec3c1cc78b89e20c2094 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Thu, 19 Sep 2024 10:55:16 -0700 Subject: [PATCH 11/11] BUG: make collection builder page use SearchTerm --- superscore/widgets/page/collection_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superscore/widgets/page/collection_builder.py b/superscore/widgets/page/collection_builder.py index f06f4bb..912d7d0 100644 --- a/superscore/widgets/page/collection_builder.py +++ b/superscore/widgets/page/collection_builder.py @@ -229,8 +229,8 @@ def add_pv(self): def update_collection_choices(self): """update collection choices based on line edit""" - search_kwargs = {'entry_type': (Collection,)} - self._coll_options = [res for res in self.client.search(**search_kwargs) + search_term = ('entry_type', 'eq', Collection) + self._coll_options = [res for res in self.client.search(search_term) if res not in (self.data.children, self)] logger.debug(f"Gathered {len(self._coll_options)} collections") self.coll_combo_box.clear()