Merge pull request #63 from fredludlow/filter-operators-rebase

Filter operators rebase
absent1706 · Aug 5, 2021 · 7d7a090 · 7d7a090
2 parents af9f3b3 + 1a20d7d
commit 7d7a090
Show file tree

Hide file tree

Showing 5 changed files with 175 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -386,6 +386,19 @@ Comment.smart_query(
 > ```
 > See [this example](examples/smartquery.py#L386)
 
+
+> ** Experimental ** 
+> Additional logic (OR, AND, NOT etc) can be expressed using a nested structure for filters, with sqlalchemy operators (or any callable) as keys:
+> ```
+> from sqlalchemy import or_
+> Comment.smart_query(filters={ or_: {
+>     'post___public': True, 
+>     'user__isnull': False
+> }})
+> ```
+> See [this example](examples/smartquery.py#L409) for more details
+
+
 ![icon](http://i.piccy.info/i9/c7168c8821f9e7023e32fd784d0e2f54/1489489664/1113/1127895/rsz_18_256.png)
 See [full example](examples/smartquery.py) and [tests](sqlalchemy_mixins/tests/test_smartquery.py)
 

diff --git a/examples/smartquery.py b/examples/smartquery.py
@@ -406,6 +406,36 @@ class Comment(BaseModel):
     schema=schema).all()
 log(res)  # cm21
 
+##### 3.2.3 Logical operators and arbitrary expressions in filters
+# If we want to use OR, NOT or other logical operators in our queries
+# we can nest the filters dictionary:
+
+res = Post.smart_query(filters={
+    sa.or_: {'archived': True, 'is_commented_by_user': u3}
+})
+log(res)   # p11, p22
+
+# Some logic cannot be expressed without using a list instead, e.g.
+# (X OR Y) AND (W OR Z)
+
+# E.g. (somewhat contrived example):
+# (non-archived OR has comments) AND 
+# (user_name like 'B%' or user_name like 'C%')
+res = Post.smart_query(filters=[
+    {sa.or_: {'archived': False, 'comments__isnull': False }},
+    {sa.or_: [
+        {'user___name__like': 'B%'},
+        {'user___name__like': 'C%'}
+    ]}
+])
+
+# !! NOTE !! This cannot be used with the where method, e.g. 
+# Post.where(**{sa.or: {...}})
+# TypeError!! (only strings are allowed as keyword arguments)
+
+# Tested with sa.or_, sa.and_ and sa._not. Other functions that
+# return a sqla expression should also work
+
 ##### 3.3 auto eager load in where() and sort() with auto-joined relations ####
 """
 Smart_query does auto-joins for filtering/sorting,

diff --git a/sqlalchemy_mixins/smartquery.py b/sqlalchemy_mixins/smartquery.py
@@ -4,7 +4,7 @@
 except ImportError:  # pragma: no cover
     pass
 
-from collections import OrderedDict
+from collections import abc, OrderedDict
 
 
 from sqlalchemy import asc, desc, inspect
@@ -24,6 +24,48 @@
 DESC_PREFIX = '-'
 
 
+def _flatten_filter_keys(filters):
+    """
+    :type filters: dict|list
+    Flatten the nested filters, extracting keys where they correspond 
+    to smart_query paths, e.g. 
+    {or_: {'id__gt': 1000, and_ : {
+        'id__lt': 500,
+        'related___property__in': (1,2,3) 
+    }}}
+    
+    Yields:
+
+    'id__gt', 'id__lt', 'related___property__in'
+
+    Also allow lists (any abc.Sequence subclass) to enable support
+    of expressions like.
+
+    (X OR Y) AND (W OR Z)
+
+    { and_: [
+        {or_: {'id__gt': 5, 'related_id__lt': 10}},
+        {or_: {'related_id2__gt': 1, 'name__like': 'Bob' }}
+    ]}
+    """
+
+    if isinstance(filters, abc.Mapping):
+        for key, value in filters.items():
+            if callable(key):
+                yield from _flatten_filter_keys(value)
+            else:
+                yield key
+
+    elif isinstance(filters, abc.Sequence):
+        for f in filters:
+            yield from _flatten_filter_keys(f)
+
+    else:
+        raise TypeError(
+            "Unsupported type (%s) in filters: %r", (type(filters), filters)
+        )
+
+
 def _parse_path_and_make_aliases(entity, entity_path, attrs, aliases):
     """
     :type entity: InspectionMixin
@@ -51,12 +93,16 @@ def _parse_path_and_make_aliases(entity, entity_path, attrs, aliases):
                 relations[relation_name] = [nested_attr]
 
     for relation_name, nested_attrs in relations.items():
-        path = entity_path + RELATION_SPLITTER + relation_name \
-               if entity_path else relation_name
+        path = (
+            entity_path + RELATION_SPLITTER + relation_name
+            if entity_path
+            else relation_name
+        )
         if relation_name not in entity.relations:
-            raise KeyError("Incorrect path `{}`: "
-                           "{} doesnt have `{}` relationship "
-                           .format(path, entity, relation_name))
+            raise KeyError(
+                "Incorrect path `{}`: "
+                "{} doesnt have `{}` relationship ".format(path, entity, relation_name)
+            )
         relationship = getattr(entity, relation_name)
         alias = aliased(relationship.property.mapper.class_)
         aliases[path] = alias, relationship
@@ -109,7 +155,7 @@ def smart_query(query, filters=None, sort_attrs=None, schema=None):
         query.session = sess
 
     root_cls = _get_root_cls(query)  # for example, User or Post
-    attrs = list(filters.keys()) + \
+    attrs = list(_flatten_filter_keys(filters)) + \
         list(map(lambda s: s.lstrip(DESC_PREFIX), sort_attrs))
     aliases = OrderedDict({})
     _parse_path_and_make_aliases(root_cls, '', attrs, aliases)
@@ -122,17 +168,28 @@ def smart_query(query, filters=None, sort_attrs=None, schema=None):
                 .options(contains_eager(relationship_path, alias=al[0]))
             loaded_paths.append(relationship_path)
 
-    for attr, value in filters.items():
-        if RELATION_SPLITTER in attr:
-            parts = attr.rsplit(RELATION_SPLITTER, 1)
-            entity, attr_name = aliases[parts[0]][0], parts[1]
-        else:
-            entity, attr_name = root_cls, attr
-        try:
-            query = query.filter(*entity.filter_expr(**{attr_name: value}))
-        except KeyError as e:
-            raise KeyError("Incorrect filter path `{}`: {}"
-                           .format(attr, e))
+    def recurse_filters(_filters):
+        if isinstance(_filters, abc.Mapping):
+            for attr, value in _filters.items():
+                if callable(attr):
+                    # E.g. or_, and_, or other sqlalchemy expression
+                    yield attr(*recurse_filters(value))
+                    continue
+                if RELATION_SPLITTER in attr:
+                    parts = attr.rsplit(RELATION_SPLITTER, 1)
+                    entity, attr_name = aliases[parts[0]][0], parts[1]
+                else:
+                    entity, attr_name = root_cls, attr
+                try:
+                    yield from entity.filter_expr(**{attr_name: value})
+                except KeyError as e:
+                    raise KeyError("Incorrect filter path `{}`: {}".format(attr, e))
+
+        elif isinstance(_filters, abc.Sequence):
+            for f in _filters:
+                yield from recurse_filters(f)
+
+    query = query.filter(*recurse_filters(filters))
 
     for attr in sort_attrs:
         if RELATION_SPLITTER in attr:

diff --git a/sqlalchemy_mixins/tests/test_smartquery.py b/sqlalchemy_mixins/tests/test_smartquery.py
@@ -528,6 +528,62 @@ def test_combinations(self):
         res = Post.where(public=False, is_commented_by_user=u1).all()
         self.assertEqual(set(res), {p11})
 
+    def test_simple_expressions(self):
+        u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+            self._seed()
+
+        res = Post.smart_query(filters={sa.or_: {'archived': True, 'is_commented_by_user': u3}}).all()
+        self.assertEqual(set(res), {p11, p22})
+
+    def test_nested_expressions(self):
+        u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+            self._seed()
+
+        # Archived posts, or (has 2016 comment rating != 1)
+        res = Post.smart_query(filters={sa.or_: {
+            'public': False,
+            sa.and_: {
+                sa.not_: {'comments___rating': 1},
+                'comments___created_at__year': 2016
+                }
+            }
+        })
+        self.assertEqual(set(res), {p11, p22})
+
+    def test_lists_in_filters_using_explicit_and(self):
+        # Check for users with (post OR comment) AND (name like 'B%' OR id>10)
+        # This cannot be expressed without a list in the filter structure
+        # (would require duplicated or_ keys)
+        u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+            self._seed()
+
+        res = User.smart_query(filters={
+            sa.and_: [
+                { sa.or_: {
+                    'comments__isnull': False,
+                    'posts__isnull': False
+                }},
+                {sa.or_: {'name__like': 'B%', 'id__gt':10}}
+            ]
+        })
+
+        self.assertEqual(set(res), {u1, u3})
+
+    def test_top_level_list_in_expression(self):
+        # Check for users with (post OR comment) AND (name like 'B%'),
+        # As above, but implicit AND
+        u1, u2, u3, p11, p12, p21, p22, cm11, cm12, cm21, cm22, cm_empty = \
+            self._seed()        
+        res = User.smart_query(filters=[
+            { sa.or_: {
+                'comments__isnull': False,
+                'posts__isnull': False
+            }},
+            {sa.or_: {'name__like': 'B%', 'id__gt':10}}
+        ])
+
+        self.assertEqual(set(res), {u1, u3})
+
 
 # noinspection PyUnusedLocal
 class TestSmartQuerySort(BaseTest):

diff --git a/sqlalchemy_mixins/tests/test_timestamp.py b/sqlalchemy_mixins/tests/test_timestamp.py
@@ -1,3 +1,4 @@
+import time
 import unittest
 import time
 from datetime import datetime