From 9f371109092f7288352a98c262b308d54ce90bac Mon Sep 17 00:00:00 2001 From: Seba Date: Mon, 23 Sep 2019 12:11:27 -0300 Subject: [PATCH 1/3] Added a n+1 startegy for paginating connections which make paginating a lot more performant avoiding the need to call on list() or length() on the iterator. We just get the desired amount of results and try to fetch one extra item. If there's a result it means there are more results, otherwhise its the end of the iteration/pagination. --- graphql_relay/connection/arrayconnection.py | 109 ++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/graphql_relay/connection/arrayconnection.py b/graphql_relay/connection/arrayconnection.py index ea951f5..4552ecc 100644 --- a/graphql_relay/connection/arrayconnection.py +++ b/graphql_relay/connection/arrayconnection.py @@ -104,6 +104,115 @@ def connection_from_list_slice(list_slice, args=None, connection_type=None, ) +def connection_from_list_slice_lazy(list_slice, args=None, connection_type=None, + edge_type=None, pageinfo_type=None, + slice_start=0, list_length=0, list_slice_length=None): + ''' + Given an iterator it consumes the needed amount based on the pagination + params, and also tries to figure out if there are more results to be + consumed. We do so by trying to fetch one more element than the specified + amount, if we were able to fetch n+1 it means there are more to be consumed. + + This spares the caller passing the total count of results, which + usually means making an extra query just to find out that number. + ''' + connection_type = connection_type or Connection + edge_type = edge_type or Edge + pageinfo_type = pageinfo_type or PageInfo + + args = args or {} + + before = args.get('before') + after = args.get('after') + first = args.get('first') + last = args.get('last') + + first_is_negative = type(first) == int and first < 0 + last_is_negative = type(last) == int and last < 0 + + start_offset = slice_start + end_offset = None + + # Validations + if first_is_negative or last_is_negative: + raise ValidationException("first and last can't be negative values") + + if first and last: + raise ValidationException( + "Including a value for both first and last is strongly discouraged," + " as it is likely to lead to confusing queries and results" + ) + + if before and last is None: + raise ValidationException( + "before without last is not supported" + ) + + if after and first is None: + raise ValidationException( + "after without first is not supported" + ) + + if before and after: + raise ValidationException( + "Mixing before and after is not supported" + ) + + if type(first) == int: + after_offset = get_offset_with_default(after, -1) + start_offset = after_offset + 1 + end_offset = start_offset + first + + elif type(last) == int: + if before: + before_offset = get_offset_with_default(before) + else: + try: + before_offset = int(list_slice.count()) + except (TypeError, AttributeError): + before_offset = len(list_slice) + end_offset = before_offset + start_offset = before_offset - last if before_offset - last > 0 else 0 + + _slice = list_slice[start_offset:end_offset] + + edges = [ + edge_type( + node=node, + cursor=offset_to_cursor(start_offset + index) + ) + for index, node in enumerate(_slice) + ] + + # To know if there are more pages we just check if there is n+1 element + # In this case it would just be the end_offset element because array indexes + # start from zero + extra_item = None + + try: + extra_item = list_slice[end_offset] + has_next_page = True + + except IndexError, TypeError: + has_next_page = False + + if extra_item is None: + has_next_page = False + + has_previous_page = start_offset > 0 + first_edge_cursor = edges[0].cursor if edges else None + last_edge_cursor = edges[-1].cursor if edges else None + + return connection_type( + edges=edges, + page_info=pageinfo_type( + start_cursor=first_edge_cursor, + end_cursor=last_edge_cursor, + has_previous_page=has_previous_page, + has_next_page=has_next_page + ) + ) + PREFIX = 'arrayconnection:' From dc37c3bae65c5ffe2e2c18afe6c6a3db0679283c Mon Sep 17 00:00:00 2001 From: Seba Date: Mon, 23 Sep 2019 12:21:10 -0300 Subject: [PATCH 2/3] Fix except error --- graphql_relay/connection/arrayconnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphql_relay/connection/arrayconnection.py b/graphql_relay/connection/arrayconnection.py index 4552ecc..f878080 100644 --- a/graphql_relay/connection/arrayconnection.py +++ b/graphql_relay/connection/arrayconnection.py @@ -193,7 +193,7 @@ def connection_from_list_slice_lazy(list_slice, args=None, connection_type=None, extra_item = list_slice[end_offset] has_next_page = True - except IndexError, TypeError: + except (IndexError, TypeError): has_next_page = False if extra_item is None: From 5e31d4b886ad91c28016c2c4f13a7393d5a1116b Mon Sep 17 00:00:00 2001 From: Seba Date: Mon, 6 Jul 2020 10:18:11 -0300 Subject: [PATCH 3/3] fix bug to avoid extra query --- graphql_relay/connection/arrayconnection.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/graphql_relay/connection/arrayconnection.py b/graphql_relay/connection/arrayconnection.py index f878080..c42dfe3 100644 --- a/graphql_relay/connection/arrayconnection.py +++ b/graphql_relay/connection/arrayconnection.py @@ -135,26 +135,26 @@ def connection_from_list_slice_lazy(list_slice, args=None, connection_type=None, # Validations if first_is_negative or last_is_negative: - raise ValidationException("first and last can't be negative values") + raise ValueError("first and last can't be negative values") if first and last: - raise ValidationException( + raise ValueError( "Including a value for both first and last is strongly discouraged," " as it is likely to lead to confusing queries and results" ) if before and last is None: - raise ValidationException( + raise ValueError( "before without last is not supported" ) if after and first is None: - raise ValidationException( + raise ValueError( "after without first is not supported" ) if before and after: - raise ValidationException( + raise ValueError( "Mixing before and after is not supported" ) @@ -174,7 +174,13 @@ def connection_from_list_slice_lazy(list_slice, args=None, connection_type=None, end_offset = before_offset start_offset = before_offset - last if before_offset - last > 0 else 0 - _slice = list_slice[start_offset:end_offset] + # Slice with an extra item to figure out if there's a next page + _slice = list_slice[start_offset : end_offset + 1] + has_next_page = False + + if (first and len(_slice) > first) or (last and len(_slice) > last): + has_next_page = True + del _slice[-1] edges = [ edge_type(