From 269798692ed73ffad41d3aec100d7ec0e796969f Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Mon, 10 Jul 2023 23:33:17 -0400 Subject: [PATCH] Add Django 4.1-4.2 and Python 3.11 support (#229) * Add Django 4.1-4.2 and Python 3.11 support (Fixes #228) * Increment version to 4.5.0 * Remove Python 2 and Django 2.2 code * Fix CI supported versions * Correct problems with tests (#234) * Fix simple error * Andrew wang/dj4.2 Improve assertNumQueries by filtering out transaction-related queries (#238) * Andrew wang/dj4.2 (#239) * Added a FilteredTransactionTestCase, updated tests. * Added more filtered tests * Removed Python <3.10 from the tox envlist for Django main. * Enhanced error message to include current database vendor of the connection for better easier troubleshooting. * Use python3.10 for GH workflows, fix postgres range issues in tests. * Check if psycopg3 is in use, if so, add psycopg3 types to CACHABLE_PARAM_TYPES. --------- Co-authored-by: Benedikt Willi --------- Co-authored-by: Jack Linke <73554672+jacklinke@users.noreply.github.com> Co-authored-by: Jack Linke Co-authored-by: Benedikt Willi Co-authored-by: Benedikt Willi --- .github/workflows/ci.yml | 20 ++++--- .github/workflows/main-ci.yml | 2 +- .gitignore | 1 + CHANGELOG.rst | 11 ++++ README.rst | 2 +- cachalot/__init__.py | 10 +--- cachalot/api.py | 36 ++++-------- cachalot/settings.py | 4 -- cachalot/tests/migrations/0001_initial.py | 16 +----- cachalot/tests/models.py | 2 +- cachalot/tests/multi_db.py | 25 +-------- cachalot/tests/postgres.py | 13 ++++- cachalot/tests/read.py | 35 +++++------- cachalot/tests/test_utils.py | 67 +++++++++++++++++++++-- cachalot/tests/thread_safety.py | 6 +- cachalot/tests/transaction.py | 31 ++++++----- cachalot/tests/write.py | 36 +++++------- cachalot/utils.py | 48 ++++++++++------ docs/index.rst | 4 +- docs/quickstart.rst | 4 +- requirements.txt | 2 +- requirements/tests.txt | 3 +- setup.py | 4 +- tox.ini | 16 +++--- 24 files changed, 215 insertions(+), 183 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 446e335f8..c0e7e2b3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,21 +6,27 @@ on: pull_request: branches: [ master ] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - django-version: ['2.2', '3.2', '4.0', '4.1'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + django-version: ['3.2', '4.1', '4.2'] exclude: - - python-version: '3.10' - django-version: '2.2' - - python-version: '3.7' - django-version: '4.0' + - python-version: '3.11' + django-version: '3.2' + - python-version: '3.11' + django-version: '4.1' - python-version: '3.7' django-version: '4.1' + - python-version: '3.7' + django-version: '4.2' services: redis: image: redis:6 @@ -71,7 +77,7 @@ jobs: ${{ matrix.python-version }}-v1- - name: Install dependencies run: | - sudo apt-get install -y libmemcached-dev zlib1g-dev + sudo apt-get install -y libmemcached-dev zlib1g-dev libpq-dev python -m pip install --upgrade pip wheel python -m pip install tox tox-gh-actions coveralls - name: Tox Test diff --git a/.github/workflows/main-ci.yml b/.github/workflows/main-ci.yml index 65368aceb..3fd636c25 100644 --- a/.github/workflows/main-ci.yml +++ b/.github/workflows/main-ci.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9'] + python-version: ['3.10'] services: redis: diff --git a/.gitignore b/.gitignore index c0c219be5..45e07f2f3 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ coverage.xml # Django stuff: *.log local_settings.py +*.sqlite3 db.sqlite3 db.sqlite3-journal diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6637bbb45..b512e9560 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,12 +1,23 @@ What’s new in django-cachalot? ============================== +2.6.0 +----- + +- Dropped Django 2.2 and 4.0 support +- Added Django 4.2 and Python 3.11 support +- Added psycopg support (#229) +- Updated tests to account for the `BEGIN` and `COMMIT` query changes in Django 4.2 +- Standardized django version comparisons in tests + 2.5.3 ----- + - Verify get_meta isn't none before requesting db_table (#225 #226) 2.5.2 ----- + - Added Django 4.1 support (#217) 2.5.1 diff --git a/README.rst b/README.rst index e58e5fdfa..1928f98de 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ Table of Contents: Quickstart ---------- -Cachalot officially supports Python 3.7-3.10 and Django 2.2, 3.2, and 4.0-4.1 with the databases PostgreSQL, SQLite, and MySQL. +Cachalot officially supports Python 3.7-3.11 and Django 3.2, 4.1, 4.2 with the databases PostgreSQL, SQLite, and MySQL. Note: an upper limit on Django version is set for your safety. Please do not ignore it. diff --git a/cachalot/__init__.py b/cachalot/__init__.py index 32a4043ea..626fea649 100644 --- a/cachalot/__init__.py +++ b/cachalot/__init__.py @@ -1,10 +1,4 @@ -VERSION = (2, 5, 3) +VERSION = (2, 6, 0) __version__ = ".".join(map(str, VERSION)) -try: - from django import VERSION as DJANGO_VERSION - - if DJANGO_VERSION < (3, 2): - default_app_config = "cachalot.apps.CachalotConfig" -except ImportError: # pragma: no cover - default_app_config = "cachalot.apps.CachalotConfig" +default_app_config = "cachalot.apps.CachalotConfig" diff --git a/cachalot/api.py b/cachalot/api.py index b4518919d..d50b277f7 100644 --- a/cachalot/api.py +++ b/cachalot/api.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +from typing import Any, Optional, Tuple, Union from django.apps import apps from django.conf import settings @@ -45,7 +46,11 @@ def _get_tables(tables_or_models): else table_or_model._meta.db_table) -def invalidate(*tables_or_models, **kwargs): +def invalidate( + *tables_or_models: Tuple[Union[str, Any], ...], + cache_alias: Optional[str] = None, + db_alias: Optional[str] = None, +) -> None: """ Clears what was cached by django-cachalot implying one or more SQL tables or models from ``tables_or_models``. @@ -62,19 +67,9 @@ def invalidate(*tables_or_models, **kwargs): (or a combination) :type tables_or_models: tuple of strings or models :arg cache_alias: Alias from the Django ``CACHES`` setting - :type cache_alias: string or NoneType :arg db_alias: Alias from the Django ``DATABASES`` setting - :type db_alias: string or NoneType :returns: Nothing - :rtype: NoneType """ - # TODO: Replace with positional arguments when we drop Python 2 support. - cache_alias = kwargs.pop('cache_alias', None) - db_alias = kwargs.pop('db_alias', None) - for k in kwargs: - raise TypeError( - "invalidate() got an unexpected keyword argument '%s'" % k) - send_signal = False invalidated = set() for cache_alias, db_alias, tables in _cache_db_tables_iterator( @@ -90,7 +85,11 @@ def invalidate(*tables_or_models, **kwargs): post_invalidation.send(table, db_alias=db_alias) -def get_last_invalidation(*tables_or_models, **kwargs): +def get_last_invalidation( + *tables_or_models: Tuple[Union[str, Any], ...], + cache_alias: Optional[str] = None, + db_alias: Optional[str] = None, +) -> float: """ Returns the timestamp of the most recent invalidation of the given ``tables_or_models``. If ``tables_or_models`` is not specified, @@ -106,19 +105,9 @@ def get_last_invalidation(*tables_or_models, **kwargs): (or a combination) :type tables_or_models: tuple of strings or models :arg cache_alias: Alias from the Django ``CACHES`` setting - :type cache_alias: string or NoneType :arg db_alias: Alias from the Django ``DATABASES`` setting - :type db_alias: string or NoneType :returns: The timestamp of the most recent invalidation - :rtype: float """ - # TODO: Replace with positional arguments when we drop Python 2 support. - cache_alias = kwargs.pop('cache_alias', None) - db_alias = kwargs.pop('db_alias', None) - for k in kwargs: - raise TypeError("get_last_invalidation() got an unexpected " - "keyword argument '%s'" % k) - last_invalidation = 0.0 for cache_alias, db_alias, tables in _cache_db_tables_iterator( list(_get_tables(tables_or_models)), cache_alias, db_alias): @@ -134,7 +123,7 @@ def get_last_invalidation(*tables_or_models, **kwargs): @contextmanager -def cachalot_disabled(all_queries=False): +def cachalot_disabled(all_queries: bool = False): """ Context manager for temporarily disabling cachalot. If you evaluate the same queryset a second time, @@ -158,7 +147,6 @@ def cachalot_disabled(all_queries=False): the original and duplicate query. :arg all_queries: Any query, including already evaluated queries, are re-evaluated. - :type all_queries: bool """ was_enabled = getattr(LOCAL_STORAGE, "cachalot_enabled", cachalot_settings.CACHALOT_ENABLED) LOCAL_STORAGE.cachalot_enabled = False diff --git a/cachalot/settings.py b/cachalot/settings.py index a7364653b..2bf5c817a 100644 --- a/cachalot/settings.py +++ b/cachalot/settings.py @@ -9,8 +9,6 @@ 'django.db.backends.sqlite3', 'django.db.backends.postgresql', 'django.db.backends.mysql', - # TODO: Remove when we drop Django 2.x support. - 'django.db.backends.postgresql_psycopg2', # GeoDjango 'django.contrib.gis.db.backends.spatialite', @@ -20,8 +18,6 @@ # django-transaction-hooks 'transaction_hooks.backends.sqlite3', 'transaction_hooks.backends.postgis', - # TODO: Remove when we drop Django 2.x support. - 'transaction_hooks.backends.postgresql_psycopg2', 'transaction_hooks.backends.mysql', # django-prometheus wrapped engines diff --git a/cachalot/tests/migrations/0001_initial.py b/cachalot/tests/migrations/0001_initial.py index 45b4f87d3..f2f97df12 100644 --- a/cachalot/tests/migrations/0001_initial.py +++ b/cachalot/tests/migrations/0001_initial.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.postgres.fields import ( ArrayField, HStoreField, IntegerRangeField, - DateRangeField, DateTimeRangeField) + DateRangeField, DateTimeRangeField, DecimalRangeField) from django.contrib.postgres.operations import ( HStoreExtension, UnaccentExtension) from django.db import models, migrations @@ -21,19 +21,6 @@ def extra_regular_available_fields(): def extra_postgres_available_fields(): fields = [] - try: - # TODO Remove when Dj31 support is dropped - from django.contrib.postgres.fields import FloatRangeField - fields.append(('float_range', FloatRangeField(null=True, blank=True))) - except ImportError: - pass - - try: - # TODO Add to module import when Dj31 is dropped - from django.contrib.postgres.fields import DecimalRangeField - fields.append(('decimal_range', DecimalRangeField(null=True, blank=True))) - except ImportError: - pass # Future proofing with Django 40 deprecation if DJANGO_VERSION[0] < 4: @@ -103,6 +90,7 @@ class Migration(migrations.Migration): ('int_range', IntegerRangeField(null=True, blank=True)), ('date_range', DateRangeField(null=True, blank=True)), ('datetime_range', DateTimeRangeField(null=True, blank=True)), + ('decimal_range', DecimalRangeField(null=True, blank=True)) ] + extra_postgres_available_fields(), ), migrations.RunSQL('CREATE TABLE cachalot_unmanagedmodel ' diff --git a/cachalot/tests/models.py b/cachalot/tests/models.py index 8c4864081..503132814 100644 --- a/cachalot/tests/models.py +++ b/cachalot/tests/models.py @@ -58,7 +58,7 @@ class PostgresModel(Model): null=True, blank=True) hstore = HStoreField(null=True, blank=True) - if DJANGO_VERSION[0] < 4: + if DJANGO_VERSION < (4, 0): from django.contrib.postgres.fields import JSONField json = JSONField(null=True, blank=True) diff --git a/cachalot/tests/multi_db.py b/cachalot/tests/multi_db.py index 8f9236216..e32ef4910 100644 --- a/cachalot/tests/multi_db.py +++ b/cachalot/tests/multi_db.py @@ -1,6 +1,5 @@ from unittest import skipIf -from django import VERSION as DJANGO_VERSION from django.conf import settings from django.db import DEFAULT_DB_ALIAS, connections, transaction from django.test import TransactionTestCase @@ -27,24 +26,6 @@ def setUp(self): # will execute an extra SQL request below. connection2.cursor() - def is_django_21_below_and_sqlite2(self): - """ - Note: See test_utils.py with this function name - Checks if Django 2.1 or below and SQLite2 - """ - django_version = DJANGO_VERSION - if not self.is_sqlite2: - # Immediately know if SQLite - return False - if django_version[0] < 2: - # Takes Django 0 and 1 out of the picture - return True - else: - if django_version[0] == 2 and django_version[1] < 2: - # Takes Django 2.0-2.1 out - return True - return False - def test_read(self): with self.assertNumQueries(1): data1 = list(Test.objects.all()) @@ -66,8 +47,7 @@ def test_invalidate_other_db(self): data1 = list(Test.objects.using(self.db_alias2)) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1, - using=self.db_alias2): + with self.assertNumQueries(1, using=self.db_alias2): t3 = Test.objects.using(self.db_alias2).create(name='test3') with self.assertNumQueries(1, using=self.db_alias2): @@ -82,8 +62,7 @@ def test_invalidation_independence(self): data1 = list(Test.objects.all()) self.assertListEqual(data1, [self.t1, self.t2]) - with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1, - using=self.db_alias2): + with self.assertNumQueries(1, using=self.db_alias2): Test.objects.using(self.db_alias2).create(name='test3') with self.assertNumQueries(0): diff --git a/cachalot/tests/postgres.py b/cachalot/tests/postgres.py index b7b18c44b..25d09a281 100644 --- a/cachalot/tests/postgres.py +++ b/cachalot/tests/postgres.py @@ -2,10 +2,21 @@ from decimal import Decimal from unittest import skipUnless +from django import VERSION from django.contrib.postgres.functions import TransactionNow from django.db import connection from django.test import TransactionTestCase, override_settings -from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange + +# If we are using Django 4.2 or higher, we need to use: +if VERSION >= (4, 2): + from django.db.backends.postgresql.psycopg_any import ( + DateRange, + DateTimeTZRange, + NumericRange, + ) +else: + from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange + from pytz import timezone from ..utils import UncachableQuery diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index f06354af1..aee07fbab 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -3,7 +3,7 @@ from uuid import UUID from decimal import Decimal -from django import VERSION as django_version +from django import VERSION as DJANGO_VERSION from django.conf import settings from django.contrib.auth.models import Group, Permission, User from django.contrib.contenttypes.models import ContentType @@ -21,7 +21,7 @@ from ..settings import cachalot_settings from ..utils import UncachableQuery from .models import Test, TestChild, TestParent, UnmanagedModel -from .test_utils import TestUtilsMixin +from .test_utils import TestUtilsMixin, FilteredTransactionTestCase from .tests_decorators import all_final_sql_checks, with_final_sql_check, no_final_sql_check @@ -36,7 +36,7 @@ def is_field_available(name): return name in fields -class ReadTestCase(TestUtilsMixin, TransactionTestCase): +class ReadTestCase(TestUtilsMixin, FilteredTransactionTestCase): """ Tests if every SQL request that only reads data is cached. @@ -353,7 +353,7 @@ def test_many_to_many_when_no_sql_check(self): @all_final_sql_checks def test_subquery(self): additional_tables = [] - if django_version[0] == 4 and django_version[1] < 1 and settings.CACHALOT_FINAL_SQL_CHECK: + if DJANGO_VERSION >= (4, 0) and DJANGO_VERSION < (4, 1) and settings.CACHALOT_FINAL_SQL_CHECK: # with Django 4.0 comes some query optimalizations that do selects little differently. additional_tables.append('django_content_type') qs = Test.objects.filter(owner__in=User.objects.all()) @@ -694,7 +694,7 @@ def _test_union(self, check: bool): self.assert_query_cached(qs) with self.assertRaisesMessage( - AssertionError if django_version[0] < 4 else TypeError, + AssertionError if DJANGO_VERSION < (4, 0) else TypeError, 'Cannot combine queries on two different base models.' ): Test.objects.all() | Permission.objects.all() @@ -739,7 +739,7 @@ def _test_intersection(self, check: bool): self.assert_query_cached(qs) with self.assertRaisesMessage( - AssertionError if django_version[0] < 4 else TypeError, + AssertionError if DJANGO_VERSION < (4, 0) else TypeError, 'Cannot combine queries on two different base models.'): Test.objects.all() & Permission.objects.all() @@ -896,7 +896,7 @@ def test_extra_order_by(self): self.assert_query_cached(qs, [self.t2, self.t1]) def test_table_inheritance(self): - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries(2): t_child = TestChild.objects.create(name='test_child') with self.assertNumQueries(1): @@ -911,15 +911,10 @@ def test_explain(self): expected = (r'\d+ 0 0 SCAN cachalot_test\n' r'\d+ 0 0 USE TEMP B-TREE FOR ORDER BY') elif self.is_mysql: - if self.django_version < (3, 1): - expected = ( - r'1 SIMPLE cachalot_test ' - r'(?:None )?ALL None None None None 2 100\.0 Using filesort') - else: - expected = ( - r'-> Sort row IDs: cachalot_test.`name` \(cost=[\d\.]+ rows=\d\)\n ' - r'-> Table scan on cachalot_test \(cost=[\d\.]+ rows=\d\)\n' - ) + expected = ( + r'-> Sort row IDs: cachalot_test.`name` \(cost=[\d\.]+ rows=\d\)\n ' + r'-> Table scan on cachalot_test \(cost=[\d\.]+ rows=\d\)\n' + ) else: explain_kwargs.update( analyze=True, @@ -935,9 +930,7 @@ def test_explain(self): r'Planning Time: [\d\.]+ ms\n' r'Execution Time: [\d\.]+ ms$') % (operation_detail, operation_detail) - with self.assertNumQueries( - 2 if self.is_mysql and django_version[0] < 3 - else 1): + with self.assertNumQueries(1): explanation1 = Test.objects.explain(**explain_kwargs) self.assertRegex(explanation1, expected) with self.assertNumQueries(0): @@ -1220,7 +1213,7 @@ def test_ipv4_address(self): @all_final_sql_checks def test_ipv6_address(self): with self.assertNumQueries(1): - test1 = Test.objects.create(name='test1', ip='2001:db8:a0b:12f0::1/64') + test1 = Test.objects.create(name='test1', ip='2001:db8:a0b:12f0::1') with self.assertNumQueries(1): test2 = Test.objects.create(name='test2', ip='2001:db8:0:85a3::ac1f:8001') @@ -1228,7 +1221,7 @@ def test_ipv6_address(self): ip__isnull=False).order_by('ip') self.assert_tables(qs, Test) self.assert_query_cached(qs, ['2001:db8:0:85a3::ac1f:8001', - '2001:db8:a0b:12f0::1/64']) + '2001:db8:a0b:12f0::1']) with self.assertNumQueries(1): Test.objects.get(ip='2001:db8:0:85a3::ac1f:8001') diff --git a/cachalot/tests/test_utils.py b/cachalot/tests/test_utils.py index d8db5f09c..7534afdb8 100644 --- a/cachalot/tests/test_utils.py +++ b/cachalot/tests/test_utils.py @@ -1,6 +1,8 @@ -from django import VERSION as DJANGO_VERSION from django.core.management.color import no_style -from django.db import connection, transaction +from django.db import DEFAULT_DB_ALIAS, connection, connections, transaction + +from django.test import TransactionTestCase +from django.test.utils import CaptureQueriesContext from ..utils import _get_tables from .models import PostgresModel @@ -11,7 +13,6 @@ def setUp(self): self.is_sqlite = connection.vendor == 'sqlite' self.is_mysql = connection.vendor == 'mysql' self.is_postgresql = connection.vendor == 'postgresql' - self.django_version = DJANGO_VERSION self.force_reopen_connection() # TODO: Remove this workaround when this issue is fixed: @@ -19,8 +20,6 @@ def setUp(self): def tearDown(self): if connection.vendor == 'postgresql': flush_args = [no_style(), (PostgresModel._meta.db_table,),] - if float(".".join(map(str, DJANGO_VERSION[:2]))) < 3.1: - flush_args.append(()) flush_sql_list = connection.ops.sql_flush(*flush_args) with transaction.atomic(): for sql in flush_sql_list: @@ -61,3 +60,61 @@ def assert_query_cached(self, queryset, result=None, result_type=None, assert_function(data2, data1) if result is not None: assert_function(data2, result) + +class FilteredTransactionTestCase(TransactionTestCase): + """ + TransactionTestCase with assertNumQueries that ignores BEGIN, COMMIT and ROLLBACK + queries. + """ + def assertNumQueries(self, num, func=None, *args, using=DEFAULT_DB_ALIAS, **kwargs): + conn = connections[using] + + context = FilteredAssertNumQueriesContext(self, num, conn) + if func is None: + return context + + with context: + func(*args, **kwargs) + + +class FilteredAssertNumQueriesContext(CaptureQueriesContext): + """ + Capture queries and assert their number ignoring BEGIN, COMMIT and ROLLBACK queries. + """ + EXCLUDE = ('BEGIN', 'COMMIT', 'ROLLBACK') + + def __init__(self, test_case, num, connection): + self.test_case = test_case + self.num = num + super().__init__(connection) + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + if exc_type is not None: + return + + filtered_queries = [] + excluded_queries = [] + for q in self.captured_queries: + if q['sql'].upper() not in self.EXCLUDE: + filtered_queries.append(q) + else: + excluded_queries.append(q) + + executed = len(filtered_queries) + + self.test_case.assertEqual( + executed, + self.num, + f"\n{executed} queries executed on {self.connection.vendor}, {self.num} expected\n" + + "\nCaptured queries were:\n" + + "".join( + f"{i}. {query['sql']}\n" + for i, query in enumerate(filtered_queries, start=1) + ) + + "\nCaptured queries, that were excluded:\n" + + "".join( + f"{i}. {query['sql']}\n" + for i, query in enumerate(excluded_queries, start=1) + ) + ) diff --git a/cachalot/tests/thread_safety.py b/cachalot/tests/thread_safety.py index cd7cd6396..cbd23dddd 100644 --- a/cachalot/tests/thread_safety.py +++ b/cachalot/tests/thread_safety.py @@ -1,10 +1,10 @@ from threading import Thread from django.db import connection, transaction -from django.test import TransactionTestCase, skipUnlessDBFeature +from django.test import skipUnlessDBFeature from .models import Test -from .test_utils import TestUtilsMixin +from .test_utils import TestUtilsMixin, FilteredTransactionTestCase class TestThread(Thread): @@ -19,7 +19,7 @@ def run(self): @skipUnlessDBFeature('test_db_allows_multiple_connections') -class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase): +class ThreadSafetyTestCase(TestUtilsMixin, FilteredTransactionTestCase): def test_concurrent_caching(self): t1 = TestThread().start_and_join() t = Test.objects.create(name='test') diff --git a/cachalot/tests/transaction.py b/cachalot/tests/transaction.py index f55f41e6c..38b371001 100644 --- a/cachalot/tests/transaction.py +++ b/cachalot/tests/transaction.py @@ -1,16 +1,17 @@ from cachalot.transaction import AtomicCache + from django.contrib.auth.models import User from django.core.cache import cache from django.db import transaction, connection, IntegrityError -from django.test import SimpleTestCase, TransactionTestCase, skipUnlessDBFeature +from django.test import SimpleTestCase, skipUnlessDBFeature from .models import Test -from .test_utils import TestUtilsMixin +from .test_utils import TestUtilsMixin, FilteredTransactionTestCase -class AtomicTestCase(TestUtilsMixin, TransactionTestCase): +class AtomicTestCase(TestUtilsMixin, FilteredTransactionTestCase): def test_successful_read_atomic(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): with transaction.atomic(): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) @@ -20,7 +21,7 @@ def test_successful_read_atomic(self): self.assertListEqual(data2, []) def test_unsuccessful_read_atomic(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): try: with transaction.atomic(): data1 = list(Test.objects.all()) @@ -38,21 +39,21 @@ def test_successful_write_atomic(self): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): with transaction.atomic(): t1 = Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(Test.objects.all()) self.assertListEqual(data2, [t1]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): with transaction.atomic(): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data3 = list(Test.objects.all()) self.assertListEqual(data3, [t1, t2]) - with self.assertNumQueries(4 if self.is_sqlite else 3): + with self.assertNumQueries(3): with transaction.atomic(): data4 = list(Test.objects.all()) t3 = Test.objects.create(name='test3') @@ -67,7 +68,7 @@ def test_unsuccessful_write_atomic(self): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): try: with transaction.atomic(): Test.objects.create(name='test') @@ -82,7 +83,7 @@ def test_unsuccessful_write_atomic(self): Test.objects.get(name='test') def test_cache_inside_atomic(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): with transaction.atomic(): data1 = list(Test.objects.all()) data2 = list(Test.objects.all()) @@ -90,7 +91,7 @@ def test_cache_inside_atomic(self): self.assertListEqual(data2, []) def test_invalidation_inside_atomic(self): - with self.assertNumQueries(4 if self.is_sqlite else 3): + with self.assertNumQueries(3): with transaction.atomic(): data1 = list(Test.objects.all()) t = Test.objects.create(name='test') @@ -99,7 +100,7 @@ def test_invalidation_inside_atomic(self): self.assertListEqual(data2, [t]) def test_successful_nested_read_atomic(self): - with self.assertNumQueries(7 if self.is_sqlite else 6): + with self.assertNumQueries(6): with transaction.atomic(): list(Test.objects.all()) with transaction.atomic(): @@ -114,7 +115,7 @@ def test_successful_nested_read_atomic(self): list(User.objects.all()) def test_unsuccessful_nested_read_atomic(self): - with self.assertNumQueries(6 if self.is_sqlite else 5): + with self.assertNumQueries(5): with transaction.atomic(): try: with transaction.atomic(): @@ -127,7 +128,7 @@ def test_unsuccessful_nested_read_atomic(self): list(Test.objects.all()) def test_successful_nested_write_atomic(self): - with self.assertNumQueries(13 if self.is_sqlite else 12): + with self.assertNumQueries(12): with transaction.atomic(): t1 = Test.objects.create(name='test1') with transaction.atomic(): @@ -144,7 +145,7 @@ def test_successful_nested_write_atomic(self): self.assertListEqual(data3, [t1, t2, t3, t4]) def test_unsuccessful_nested_write_atomic(self): - with self.assertNumQueries(16 if self.is_sqlite else 15): + with self.assertNumQueries(15): with transaction.atomic(): t1 = Test.objects.create(name='test1') try: diff --git a/cachalot/tests/write.py b/cachalot/tests/write.py index 982bb6368..dd6459af9 100644 --- a/cachalot/tests/write.py +++ b/cachalot/tests/write.py @@ -1,6 +1,5 @@ from unittest import skipIf, skipUnless -from django import VERSION as DJANGO_VERSION from django.contrib.auth.models import User, Permission, Group from django.contrib.contenttypes.models import ContentType from django.core.exceptions import MultipleObjectsReturned @@ -12,10 +11,10 @@ from django.test import TransactionTestCase, skipUnlessDBFeature from .models import Test, TestParent, TestChild -from .test_utils import TestUtilsMixin +from .test_utils import TestUtilsMixin, FilteredTransactionTestCase -class WriteTestCase(TestUtilsMixin, TransactionTestCase): +class WriteTestCase(TestUtilsMixin, FilteredTransactionTestCase): """ Tests if every SQL request writing data is not cached and invalidates the implied data. @@ -56,7 +55,7 @@ def test_get_or_create(self): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries(2): t, created = Test.objects.get_or_create(name='test') self.assertTrue(created) @@ -78,14 +77,14 @@ def test_update_or_create(self): with self.assertNumQueries(1): self.assertListEqual(list(Test.objects.all()), []) - with self.assertNumQueries(5 if self.is_sqlite else 4): + with self.assertNumQueries(4): t, created = Test.objects.update_or_create( name='test', defaults={'public': True}) self.assertTrue(created) self.assertEqual(t.name, 'test') self.assertEqual(t.public, True) - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries(2): t, created = Test.objects.update_or_create( name='test', defaults={'public': False}) self.assertFalse(created) @@ -94,7 +93,7 @@ def test_update_or_create(self): # The number of SQL queries doesn’t decrease because update_or_create # always calls an UPDATE, even when data wasn’t changed. - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries(2): t, created = Test.objects.update_or_create( name='test', defaults={'public': False}) self.assertFalse(created) @@ -109,12 +108,12 @@ def test_bulk_create(self): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)] Test.objects.bulk_create(unsaved_tests) self.assertEqual(Test.objects.count(), 10) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)] Test.objects.bulk_create(unsaved_tests) self.assertEqual(Test.objects.count(), 20) @@ -160,12 +159,12 @@ def test_delete(self): self.assertListEqual(data1, [t1.name, t2.name]) self.assertListEqual(data2, [t1.name]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): Test.objects.bulk_create([Test(name='test%s' % i) for i in range(2, 11)]) with self.assertNumQueries(1): self.assertEqual(Test.objects.count(), 10) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): Test.objects.all().delete() with self.assertNumQueries(1): self.assertEqual(Test.objects.count(), 0) @@ -360,7 +359,7 @@ def test_invalidate_annotate(self): self.assertListEqual(data4, [user1, user2]) self.assertListEqual([u.n for u in data4], [1, 0]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): Test.objects.bulk_create([ Test(name='test3', owner=user1), Test(name='test4', owner=user2), @@ -588,7 +587,7 @@ def test_invalidate_select_related(self): data2 = list(Test.objects.select_related('owner')) self.assertListEqual(data2, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): Test.objects.bulk_create([ Test(name='test1', owner=u1), Test(name='test2', owner=u2), @@ -602,7 +601,7 @@ def test_invalidate_select_related(self): self.assertEqual(data3[2].owner, u2) self.assertEqual(data3[3].owner, u1) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(1): Test.objects.filter(name__in=['test1', 'test2']).delete() with self.assertNumQueries(1): data4 = list(Test.objects.select_related('owner')) @@ -634,12 +633,7 @@ def test_invalidate_prefetch_related(self): self.assertEqual(data3[0].owner, u) self.assertListEqual(list(data3[0].owner.groups.all()), []) - with self.assertNumQueries( - 8 if self.is_sqlite and DJANGO_VERSION[0] == 2 and DJANGO_VERSION[1] == 2 - else 4 if self.is_postgresql and DJANGO_VERSION[0] > 2 - else 4 if self.is_mysql and DJANGO_VERSION[0] > 2 - else 6 - ): + with self.assertNumQueries(4): group = Group.objects.create(name='test_group') permissions = list(Permission.objects.all()[:5]) group.permissions.add(*permissions) @@ -852,7 +846,7 @@ def test_invalidate_table_inheritance(self): with self.assertRaises(TestChild.DoesNotExist): TestChild.objects.get() - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries(2): t_child = TestChild.objects.create(name='test_child') with self.assertNumQueries(1): diff --git a/cachalot/utils.py b/cachalot/utils.py index 9df2a74ec..3c2d6ed04 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -6,7 +6,7 @@ from uuid import UUID from django.contrib.postgres.functions import TransactionNow -from django.db import connections +from django.db import connections, connection from django.db.models import Exists, QuerySet, Subquery from django.db.models.expressions import RawSQL from django.db.models.functions import Now @@ -35,24 +35,42 @@ class IsRawQuery(Exception): } UNCACHABLE_FUNCS = {Now, TransactionNow} -try: - # TODO Drop after Dj30 drop - from django.contrib.postgres.fields.jsonb import JsonAdapter - CACHABLE_PARAM_TYPES.update((JsonAdapter,)) -except ImportError: - pass +# Check if psycopg2 or 3 is used +connection.ensure_connection() +underlying_connection = connection.connection -try: +if underlying_connection.__class__.__module__.startswith('psycopg2'): from psycopg2 import Binary from psycopg2.extras import ( NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json) -except ImportError: - pass -else: CACHABLE_PARAM_TYPES.update(( Binary, NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json,)) +elif underlying_connection.__class__.__module__.startswith('psycopg'): + from django.db.backends.postgresql.psycopg_any import ( + NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, + ) + + from psycopg.dbapi20 import Binary + + from psycopg.types.numeric import ( + Int2, Int4, Int8, Float4, Float8, + ) + + from ipaddress import ( + IPv4Address, + IPv6Address, + ) + from psycopg.types.json import ( + Json, Jsonb, + ) + + CACHABLE_PARAM_TYPES.update(( + NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json, Jsonb, + Int2, Int4, Int8, Float4, Float8, IPv4Address, IPv6Address, + Binary, + )) def check_parameter_types(params): for p in params: @@ -131,13 +149,7 @@ def _find_rhs_lhs_subquery(side): elif h_class is QuerySet: return side.query elif h_class in (Subquery, Exists): # Subquery allows QuerySet & Query - try: - return side.query.query if side.query.__class__ is QuerySet else side.query - except AttributeError: # TODO Remove try/except closure after drop Django 2.2 - try: - return side.queryset.query - except AttributeError: - return None + return side.query.query if side.query.__class__ is QuerySet else side.query elif h_class in UNCACHABLE_FUNCS: raise UncachableQuery diff --git a/docs/index.rst b/docs/index.rst index fedf53f7d..46051d0b4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,8 +11,8 @@ Caches your Django ORM queries and automatically invalidates them. .. image:: http://img.shields.io/pypi/v/django-cachalot.svg?style=flat-square&maxAge=3600 :target: https://pypi.python.org/pypi/django-cachalot -.. image:: http://img.shields.io/travis/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600 - :target: https://travis-ci.org/noripyt/django-cachalot +.. image:: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml/badge.svg + :target: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml .. image:: http://img.shields.io/coveralls/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600 :target: https://coveralls.io/r/noripyt/django-cachalot?branch=master diff --git a/docs/quickstart.rst b/docs/quickstart.rst index b2710018c..980d775fa 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,8 +4,8 @@ Quick start Requirements ............ -- Django 2.2, 3.2, 4.0-4.1 -- Python 3.7-3.10 +- Django 3.2, 4.1, 4.2 +- Python 3.7-3.11 - a cache configured as ``'default'`` with one of these backends: - `django-redis `_ diff --git a/requirements.txt b/requirements.txt index 7613f70f9..316ec4330 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Django>=2.2,<4.2 +Django>=3.2,<4.3 diff --git a/requirements/tests.txt b/requirements/tests.txt index 4e1726087..85032677f 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -2,7 +2,8 @@ django>=2 -psycopg2-binary +psycopg2 +psycopg mysqlclient django-redis python-memcached diff --git a/setup.py b/setup.py index 4bcde17ee..a788af3eb 100755 --- a/setup.py +++ b/setup.py @@ -25,15 +25,15 @@ 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', - 'Framework :: Django :: 2.2', 'Framework :: Django :: 3.2', - 'Framework :: Django :: 4.0', 'Framework :: Django :: 4.1', + 'Framework :: Django :: 4.2', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Internet :: WWW/HTTP', ], license='BSD', diff --git a/tox.ini b/tox.ini index 166b3597c..f76099c1e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] envlist = - py{37,38,39}-django2.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{37,38,39,310}-django3.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{38,39,310}-django4.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{38,39,310}-django4.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{38,39,310}-djangomain-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{38,39,310,311}-django4.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{310,311}-djangomain-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, [testenv] passenv = * @@ -13,13 +12,14 @@ basepython = py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 deps = - django2.2: Django>=2.2,<2.3 django3.2: Django>=3.2,<4.0 - django4.0: Django>=4.0,<4.1 django4.1: Django>=4.1,<4.2 + django4.2: Django>=4.2,<4.3 djangomain: https://github.com/django/django/archive/main.tar.gz - psycopg2-binary>=2.8,<2.9 + psycopg2>=2.9.5,<3.0 + psycopg mysqlclient django-redis python-memcached @@ -48,11 +48,11 @@ python = 3.8: py38 3.9: py39 3.10: py310 + 3.11: py311 [gh-actions:env] DJANGO = - 2.2: django2.2 3.2: django3.2 - 4.0: django4.0 4.1: django4.1 + 4.2: django4.2 main: djangomain