From d30430d354ebbc8523c0fe8326b5c82fa31c8e6a Mon Sep 17 00:00:00 2001 From: Fred Ludlow Date: Thu, 15 Oct 2020 22:35:08 +0100 Subject: [PATCH 1/2] Allow arbitrary sqla operators in smart_query filters --- README.md | 14 +++++- examples/smartquery.py | 16 +++++++ sqlalchemy_mixins/smartquery.py | 55 +++++++++++++++++----- sqlalchemy_mixins/tests/test_smartquery.py | 22 +++++++++ sqlalchemy_mixins/tests/test_timestamp.py | 1 + 5 files changed, 95 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 43e0d68..cde913c 100644 --- a/README.md +++ b/README.md @@ -389,6 +389,18 @@ Comment.smart_query( > ``` > See [this example](examples/smartquery.py#L386) + +> ** Experimental ** +> Additional logic (OR, AND, NOT etc) can be expressed using a nested dictionary for filters, with sqlalchemy operators (or any callable) as keys: +> ``` +> from sqlalchemy import or_ +> Comment.smart_query(filters={ or_: { +> 'post___public': True, +> 'user__isnull': False +> }}) +> ``` +> See [this example](examples/smartquery.py#L409) + ![icon](http://i.piccy.info/i9/c7168c8821f9e7023e32fd784d0e2f54/1489489664/1113/1127895/rsz_18_256.png) See [full example](examples/smartquery.py) and [tests](sqlalchemy_mixins/tests/test_smartquery.py) @@ -671,4 +683,4 @@ removed [TimestampsMixin](#timestamps) from [AllFeaturesMixin](sqlalchemy_mixins ### v1.3 -Add support for SQLAlchemy 1.4 \ No newline at end of file +Add support for SQLAlchemy 1.4 diff --git a/examples/smartquery.py b/examples/smartquery.py index 4d9c34b..ed2cb8a 100644 --- a/examples/smartquery.py +++ b/examples/smartquery.py @@ -406,6 +406,22 @@ class Comment(BaseModel): schema=schema).all() log(res) # cm21 +##### 3.2.3 Logical operators and arbitrary expressions in filters +# If we want to use OR, NOT or other logical operators in our queries +# we can nest the filters dictionary: + +res = Post.smart_query(filters={ + sa.or_: {'archived': True, 'is_commented_by_user': u3} +}) +log(res) # p11, p22 + +# !! NOTE !! This cannot be used with the where method, e.g. +# Post.where(**{sa.or: {...}}) +# TypeError!! (only strings are allowed as keyword arguments) + +# sa.or_, sa.and_ and sa._not work, as should all functions that +# return a sqla expression. + ##### 3.3 auto eager load in where() and sort() with auto-joined relations #### """ Smart_query does auto-joins for filtering/sorting, diff --git a/sqlalchemy_mixins/smartquery.py b/sqlalchemy_mixins/smartquery.py index 42aa2cb..500f737 100644 --- a/sqlalchemy_mixins/smartquery.py +++ b/sqlalchemy_mixins/smartquery.py @@ -24,6 +24,30 @@ DESC_PREFIX = '-' +def _flatten_filter_keys(filters): + """ + :type filters: dict + Flatten the nested filter dictionaries, discarding callables. Sample input: + { + or_: { + 'id__gt': 1000, and_ : { + 'id__lt': 500, 'related___property__in': (1,2,3) + } + } + } + + Yields: + + 'id__gt', 'id__lt', 'related___property__in' + + """ + for key, value in filters.items(): + if callable(key): + yield from _flatten_filter_keys(value) + else: + yield key + + def _parse_path_and_make_aliases(entity, entity_path, attrs, aliases): """ :type entity: InspectionMixin @@ -104,7 +128,7 @@ def smart_query(query, filters=None, sort_attrs=None, schema=None): query.session = sess root_cls = _get_root_cls(query) # for example, User or Post - attrs = list(filters.keys()) + \ + attrs = list(_flatten_filter_keys(filters)) + \ list(map(lambda s: s.lstrip(DESC_PREFIX), sort_attrs)) aliases = OrderedDict({}) _parse_path_and_make_aliases(root_cls, '', attrs, aliases) @@ -116,17 +140,24 @@ def smart_query(query, filters=None, sort_attrs=None, schema=None): .options(contains_eager(relationship_path, alias=al[0])) loaded_paths.append(relationship_path) - for attr, value in filters.items(): - if RELATION_SPLITTER in attr: - parts = attr.rsplit(RELATION_SPLITTER, 1) - entity, attr_name = aliases[parts[0]][0], parts[1] - else: - entity, attr_name = root_cls, attr - try: - query = query.filter(*entity.filter_expr(**{attr_name: value})) - except KeyError as e: - raise KeyError("Incorrect filter path `{}`: {}" - .format(attr, e)) + def recurse_filters(_filters): + for attr, value in _filters.items(): + if callable(attr): + # E.g. or_, and_, or other sqlalchemy function + # that returns an expression + yield attr(*recurse_filters(value)) + continue + if RELATION_SPLITTER in attr: + parts = attr.rsplit(RELATION_SPLITTER, 1) + entity, attr_name = aliases[parts[0]][0], parts[1] + else: + entity, attr_name = root_cls, attr + try: + yield from entity.filter_expr(**{attr_name: value}) + except KeyError as e: + raise KeyError("Incorrect filter path `{}`: {}".format(attr, e)) + + query = query.filter(*recurse_filters(filters)) for attr in sort_attrs: if RELATION_SPLITTER in attr: diff --git a/sqlalchemy_mixins/tests/test_smartquery.py b/sqlalchemy_mixins/tests/test_smartquery.py index c45f4f6..ce4e2d7 100644 --- a/sqlalchemy_mixins/tests/test_smartquery.py +++ b/sqlalchemy_mixins/tests/test_smartquery.py @@ -528,6 +528,28 @@ def test_combinations(self): res = Post.where(public=False, is_commented_by_user=u1).all() self.assertEqual(set(res), {p11}) + def test_simple_expressions(self): + u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \ + self._seed() + + res = Post.smart_query(filters={sa.or_: {'archived': True, 'is_commented_by_user': u3}}).all() + self.assertEqual(set(res), {p11, p22}) + + def test_nested_expressions(self): + u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \ + self._seed() + + # Archived posts, or (has 2016 comment rating != 1) + res = Post.smart_query(filters={sa.or_: { + 'public': False, + sa.and_: { + sa.not_: {'comments___rating': 1}, + 'comments___created_at__year': 2016 + } + } + }) + self.assertEqual(set(res), {p11, p22}) + # noinspection PyUnusedLocal class TestSmartQuerySort(BaseTest): diff --git a/sqlalchemy_mixins/tests/test_timestamp.py b/sqlalchemy_mixins/tests/test_timestamp.py index 3976a50..3aef2a2 100644 --- a/sqlalchemy_mixins/tests/test_timestamp.py +++ b/sqlalchemy_mixins/tests/test_timestamp.py @@ -1,3 +1,4 @@ +import time import unittest import time from datetime import datetime From 1a20d7d5c35fa3c5033d862b234980e4746f31e4 Mon Sep 17 00:00:00 2001 From: Fred Ludlow Date: Tue, 20 Oct 2020 10:48:45 +0100 Subject: [PATCH 2/2] Extend smart_query filter syntax to allow lists (tuples etc). for repeating conditions --- README.md | 5 +- examples/smartquery.py | 18 +++- sqlalchemy_mixins/smartquery.py | 96 ++++++++++++++-------- sqlalchemy_mixins/tests/test_smartquery.py | 34 ++++++++ 4 files changed, 114 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index cde913c..2c75f62 100644 --- a/README.md +++ b/README.md @@ -391,7 +391,7 @@ Comment.smart_query( > ** Experimental ** -> Additional logic (OR, AND, NOT etc) can be expressed using a nested dictionary for filters, with sqlalchemy operators (or any callable) as keys: +> Additional logic (OR, AND, NOT etc) can be expressed using a nested structure for filters, with sqlalchemy operators (or any callable) as keys: > ``` > from sqlalchemy import or_ > Comment.smart_query(filters={ or_: { @@ -399,7 +399,8 @@ Comment.smart_query( > 'user__isnull': False > }}) > ``` -> See [this example](examples/smartquery.py#L409) +> See [this example](examples/smartquery.py#L409) for more details + ![icon](http://i.piccy.info/i9/c7168c8821f9e7023e32fd784d0e2f54/1489489664/1113/1127895/rsz_18_256.png) See [full example](examples/smartquery.py) and [tests](sqlalchemy_mixins/tests/test_smartquery.py) diff --git a/examples/smartquery.py b/examples/smartquery.py index ed2cb8a..550b430 100644 --- a/examples/smartquery.py +++ b/examples/smartquery.py @@ -415,12 +415,26 @@ class Comment(BaseModel): }) log(res) # p11, p22 +# Some logic cannot be expressed without using a list instead, e.g. +# (X OR Y) AND (W OR Z) + +# E.g. (somewhat contrived example): +# (non-archived OR has comments) AND +# (user_name like 'B%' or user_name like 'C%') +res = Post.smart_query(filters=[ + {sa.or_: {'archived': False, 'comments__isnull': False }}, + {sa.or_: [ + {'user___name__like': 'B%'}, + {'user___name__like': 'C%'} + ]} +]) + # !! NOTE !! This cannot be used with the where method, e.g. # Post.where(**{sa.or: {...}}) # TypeError!! (only strings are allowed as keyword arguments) -# sa.or_, sa.and_ and sa._not work, as should all functions that -# return a sqla expression. +# Tested with sa.or_, sa.and_ and sa._not. Other functions that +# return a sqla expression should also work ##### 3.3 auto eager load in where() and sort() with auto-joined relations #### """ diff --git a/sqlalchemy_mixins/smartquery.py b/sqlalchemy_mixins/smartquery.py index 500f737..737a6a6 100644 --- a/sqlalchemy_mixins/smartquery.py +++ b/sqlalchemy_mixins/smartquery.py @@ -4,7 +4,7 @@ except ImportError: # pragma: no cover pass -from collections import OrderedDict +from collections import abc, OrderedDict import sqlalchemy from sqlalchemy import asc, desc, inspect @@ -26,26 +26,44 @@ def _flatten_filter_keys(filters): """ - :type filters: dict - Flatten the nested filter dictionaries, discarding callables. Sample input: - { - or_: { - 'id__gt': 1000, and_ : { - 'id__lt': 500, 'related___property__in': (1,2,3) - } - } - } + :type filters: dict|list + Flatten the nested filters, extracting keys where they correspond + to smart_query paths, e.g. + {or_: {'id__gt': 1000, and_ : { + 'id__lt': 500, + 'related___property__in': (1,2,3) + }}} Yields: 'id__gt', 'id__lt', 'related___property__in' + Also allow lists (any abc.Sequence subclass) to enable support + of expressions like. + + (X OR Y) AND (W OR Z) + + { and_: [ + {or_: {'id__gt': 5, 'related_id__lt': 10}}, + {or_: {'related_id2__gt': 1, 'name__like': 'Bob' }} + ]} """ - for key, value in filters.items(): - if callable(key): - yield from _flatten_filter_keys(value) - else: - yield key + + if isinstance(filters, abc.Mapping): + for key, value in filters.items(): + if callable(key): + yield from _flatten_filter_keys(value) + else: + yield key + + elif isinstance(filters, abc.Sequence): + for f in filters: + yield from _flatten_filter_keys(f) + + else: + raise TypeError( + "Unsupported type (%s) in filters: %r", (type(filters), filters) + ) def _parse_path_and_make_aliases(entity, entity_path, attrs, aliases): @@ -75,12 +93,16 @@ def _parse_path_and_make_aliases(entity, entity_path, attrs, aliases): relations[relation_name] = [nested_attr] for relation_name, nested_attrs in relations.items(): - path = entity_path + RELATION_SPLITTER + relation_name \ - if entity_path else relation_name + path = ( + entity_path + RELATION_SPLITTER + relation_name + if entity_path + else relation_name + ) if relation_name not in entity.relations: - raise KeyError("Incorrect path `{}`: " - "{} doesnt have `{}` relationship " - .format(path, entity, relation_name)) + raise KeyError( + "Incorrect path `{}`: " + "{} doesnt have `{}` relationship ".format(path, entity, relation_name) + ) relationship = getattr(entity, relation_name) alias = aliased(relationship.property.mapper.class_) aliases[path] = alias, relationship @@ -141,21 +163,25 @@ def smart_query(query, filters=None, sort_attrs=None, schema=None): loaded_paths.append(relationship_path) def recurse_filters(_filters): - for attr, value in _filters.items(): - if callable(attr): - # E.g. or_, and_, or other sqlalchemy function - # that returns an expression - yield attr(*recurse_filters(value)) - continue - if RELATION_SPLITTER in attr: - parts = attr.rsplit(RELATION_SPLITTER, 1) - entity, attr_name = aliases[parts[0]][0], parts[1] - else: - entity, attr_name = root_cls, attr - try: - yield from entity.filter_expr(**{attr_name: value}) - except KeyError as e: - raise KeyError("Incorrect filter path `{}`: {}".format(attr, e)) + if isinstance(_filters, abc.Mapping): + for attr, value in _filters.items(): + if callable(attr): + # E.g. or_, and_, or other sqlalchemy expression + yield attr(*recurse_filters(value)) + continue + if RELATION_SPLITTER in attr: + parts = attr.rsplit(RELATION_SPLITTER, 1) + entity, attr_name = aliases[parts[0]][0], parts[1] + else: + entity, attr_name = root_cls, attr + try: + yield from entity.filter_expr(**{attr_name: value}) + except KeyError as e: + raise KeyError("Incorrect filter path `{}`: {}".format(attr, e)) + + elif isinstance(_filters, abc.Sequence): + for f in _filters: + yield from recurse_filters(f) query = query.filter(*recurse_filters(filters)) diff --git a/sqlalchemy_mixins/tests/test_smartquery.py b/sqlalchemy_mixins/tests/test_smartquery.py index ce4e2d7..995fde4 100644 --- a/sqlalchemy_mixins/tests/test_smartquery.py +++ b/sqlalchemy_mixins/tests/test_smartquery.py @@ -550,6 +550,40 @@ def test_nested_expressions(self): }) self.assertEqual(set(res), {p11, p22}) + def test_lists_in_filters_using_explicit_and(self): + # Check for users with (post OR comment) AND (name like 'B%' OR id>10) + # This cannot be expressed without a list in the filter structure + # (would require duplicated or_ keys) + u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \ + self._seed() + + res = User.smart_query(filters={ + sa.and_: [ + { sa.or_: { + 'comments__isnull': False, + 'posts__isnull': False + }}, + {sa.or_: {'name__like': 'B%', 'id__gt':10}} + ] + }) + + self.assertEqual(set(res), {u1, u3}) + + def test_top_level_list_in_expression(self): + # Check for users with (post OR comment) AND (name like 'B%'), + # As above, but implicit AND + u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \ + self._seed() + res = User.smart_query(filters=[ + { sa.or_: { + 'comments__isnull': False, + 'posts__isnull': False + }}, + {sa.or_: {'name__like': 'B%', 'id__gt':10}} + ]) + + self.assertEqual(set(res), {u1, u3}) + # noinspection PyUnusedLocal class TestSmartQuerySort(BaseTest):