diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index b661b9382..f1637505e 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -9,28 +9,23 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.9, 3.10.0-rc.1] steps: - name: Checkout the repository uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install package run: | - pip install flake8==3.7.7 - pip install mypy==0.720 - pip install mypy-zope==0.2.0 - pip install black==19.10b0 - pip install isort==4.3.21 + pip install $(grep -P 'flake8|black|isort' contrib-requirements.txt) - name: Run pre-checks run: | flake8 guillotina --config=setup.cfg - mypy guillotina/ --ignore-missing-imports isort -c -rc guillotina/ black --check --verbose guillotina # Job to run tests @@ -39,7 +34,7 @@ jobs: strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.9, 3.10.0-rc.1] database: ["DUMMY", "postgres", "cockroachdb"] db_schema: ["custom", "public"] exclude: @@ -59,7 +54,7 @@ jobs: uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -74,6 +69,10 @@ jobs: - name: Start memcached image uses: niden/actions-memcached@v7 + - name: Check mypy + run: | + mypy guillotina/ + - name: Run tests run: | pytest -rfE --reruns 2 --cov=guillotina -s --tb=native -v --cov-report xml --cov-append guillotina diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5186c1b80..89554b591 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,16 +1,23 @@ CHANGELOG ========= -6.3.16 (unreleased) +6.4.0 (unreleased) ------------------- +- Add support for Python 3.10 + [masipcat] +- Reimplemented IRedisUtility to adapt to aioredis v2.0 breaking changes + [masipcat] +- Upgraded dev dependencies + [masipcat] +- Use custom implementation of ContextVar that is not copied automatically to new tasks + [masipcat] - Fix vocabulray country code AN - 6.3.15 (2021-08-05) ------------------- -- fix: Add MIMEMultipart('alternative') to attach message in parent MIMEMultipart to render only html body. +- fix: Add MIMEMultipart('alternative') to attach message in parent MIMEMultipart to render only html body. [rboixaderg] 6.3.14 (2021-08-04) diff --git a/contrib-requirements.txt b/contrib-requirements.txt index 7725bcf14..08e4020af 100644 --- a/contrib-requirements.txt +++ b/contrib-requirements.txt @@ -1,15 +1,14 @@ -aioredis==1.3.1 +aioredis==2.0.0 html2text==2019.8.11 aiosmtplib==1.1.4 pre-commit==1.18.2 -flake8==3.7.7 +flake8==3.9.2 codecov==2.0.15 -mypy==0.720 -mypy-zope==0.2.0 +mypy-zope==0.3.2 black==19.10b0 isort==4.3.21 jinja2==2.11.3 pytz==2020.1 -emcache==0.4.1 -pymemcache==3.4.0 -pillow==8.2.0 \ No newline at end of file +emcache==0.4.1; python_version < '3.10' +pymemcache==3.4.0; python_version < '3.10' +pillow==8.3.2 diff --git a/guillotina/api/login.py b/guillotina/api/login.py index fdbce7c5c..c7cdbd205 100644 --- a/guillotina/api/login.py +++ b/guillotina/api/login.py @@ -109,7 +109,7 @@ async def __call__(self): jwt_token = jwt.encode( data, app_settings["jwt"]["secret"], algorithm=app_settings["jwt"]["algorithm"] - ).decode("utf-8") + ) await notify(UserRefreshToken(user, jwt_token)) diff --git a/guillotina/async_util.py b/guillotina/async_util.py index 5281380de..c5d44098d 100644 --- a/guillotina/async_util.py +++ b/guillotina/async_util.py @@ -1,4 +1,3 @@ -from dateutil.tz import tzutc from guillotina import logger from guillotina import task_vars from guillotina.db.transaction import Status @@ -18,9 +17,6 @@ import typing -_zone = tzutc() - - class QueueUtility(object): def __init__(self, settings=None, loop=None): self._queue = None @@ -31,7 +27,7 @@ def __init__(self, settings=None, loop=None): @property def queue(self): if self._queue is None: - self._queue = asyncio.Queue(loop=self._loop) + self._queue = asyncio.Queue() return self._queue async def initialize(self, app=None): diff --git a/guillotina/auth/utils.py b/guillotina/auth/utils.py index c7e36410a..13f82a5d0 100644 --- a/guillotina/auth/utils.py +++ b/guillotina/auth/utils.py @@ -61,7 +61,5 @@ def authenticate_user(userid, data=None, timeout=60 * 60 * 1): "sub": userid, } ) - jwt_token = jwt.encode( - data, app_settings["jwt"]["secret"], algorithm=app_settings["jwt"]["algorithm"] - ).decode("utf-8") + jwt_token = jwt.encode(data, app_settings["jwt"]["secret"], algorithm=app_settings["jwt"]["algorithm"]) return jwt_token, data diff --git a/guillotina/commands/__init__.py b/guillotina/commands/__init__.py index ed1d86bf8..1a5cd2ef1 100644 --- a/guillotina/commands/__init__.py +++ b/guillotina/commands/__init__.py @@ -218,7 +218,7 @@ async def cleanup(self, app): await app.shutdown() except Exception: logger.warning("Unhandled error cleanup tasks", exc_info=True) - for task in asyncio.Task.all_tasks(): + for task in asyncio.all_tasks(): if task.done(): continue if "cleanup" in task._coro.__qualname__: diff --git a/guillotina/component/globalregistry.py b/guillotina/component/globalregistry.py index 1d086d116..5e026fb2a 100644 --- a/guillotina/component/globalregistry.py +++ b/guillotina/component/globalregistry.py @@ -129,7 +129,7 @@ def __init__(self, parent, name): @implementer(IComponentLookup) -class GlobalComponents(Components): +class GlobalComponents(Components): # type: ignore def _init_registries(self): self.adapters = GuillotinaAdapterRegistry(self, "adapters") self.utilities = GuillotinaAdapterRegistry(self, "utilities") diff --git a/guillotina/configure/config.py b/guillotina/configure/config.py index 76fda4a28..0e46f98c7 100644 --- a/guillotina/configure/config.py +++ b/guillotina/configure/config.py @@ -133,7 +133,7 @@ def __init__(self, etype, evalue): self.etype, self.evalue = etype, evalue def __str__(self): # pragma NO COVER - return "%s: %s\n in:\n %s" % (self.etype, self.evalue) + return "%s: %s" % (self.etype, self.evalue) ############################################################################## diff --git a/guillotina/contrib/cache/strategy.py b/guillotina/contrib/cache/strategy.py index e16a9fcf7..760e841d8 100644 --- a/guillotina/contrib/cache/strategy.py +++ b/guillotina/contrib/cache/strategy.py @@ -8,6 +8,7 @@ from guillotina.exceptions import NoPubSubUtility from guillotina.interfaces import ICacheUtility from guillotina.profile import profilable +from guillotina.task_vars import copy_context from typing import Any from typing import Dict from typing import List @@ -129,7 +130,7 @@ async def close(self, invalidate=True, publish=True): await self.fill_cache() if len(self._keys_to_publish) > 0 and self._utility._subscriber is not None: keys = self._keys_to_publish - asyncio.ensure_future(self.synchronize(keys)) + asyncio.ensure_future(copy_context(self.synchronize(keys))) else: self._stored_objects.clear() else: diff --git a/guillotina/contrib/mailer/utility.py b/guillotina/contrib/mailer/utility.py index 1f61d339a..bac71da7c 100644 --- a/guillotina/contrib/mailer/utility.py +++ b/guillotina/contrib/mailer/utility.py @@ -9,6 +9,7 @@ from guillotina.contrib.mailer.exceptions import NoEndpointDefinedException from guillotina.interfaces import IMailEndpoint from guillotina.interfaces import IMailer +from guillotina.task_vars import copy_context from guillotina.utils import get_random_string from zope.interface import implementer @@ -135,7 +136,7 @@ def get_endpoint(self, endpoint_name): else: raise NoEndpointDefinedException("{} mail endpoint not defined".format(endpoint_name)) utility.from_settings(settings) - asyncio.ensure_future(utility.initialize()) + asyncio.ensure_future(copy_context(utility.initialize())) self._endpoints[endpoint_name] = utility return self._endpoints[endpoint_name] @@ -223,7 +224,7 @@ async def finalize(self): @implementer(IMailer) class PrintingMailerUtility(MailerUtility): def __init__(self, settings=None, loop=None): - self._queue = asyncio.Queue(loop=loop) + self._queue = asyncio.Queue() self._settings = settings or {} async def _send(self, sender, recipients, message, endpoint_name="default"): @@ -233,7 +234,7 @@ async def _send(self, sender, recipients, message, endpoint_name="default"): @implementer(IMailer) class TestMailerUtility(MailerUtility): def __init__(self, settings=None, loop=None): - self._queue = asyncio.Queue(loop=loop) + self._queue = asyncio.Queue() self.mail = [] async def send( diff --git a/guillotina/contrib/pubsub/utility.py b/guillotina/contrib/pubsub/utility.py index ab3020b02..1e3680301 100644 --- a/guillotina/contrib/pubsub/utility.py +++ b/guillotina/contrib/pubsub/utility.py @@ -55,10 +55,9 @@ async def finalize(self, app): self._initialized = False await asyncio.sleep(0.1) - async def real_subscribe(self, channel_name): + async def real_subscribe(self, channel, channel_name): while channel_name in self._subscribers: try: - channel = await self._driver.subscribe(channel_name) async for msg in channel: try: try: @@ -76,10 +75,14 @@ async def real_subscribe(self, channel_name): return except Exception: logger.error(f"Unhandled exception with pubsub. Sleeping before trying again", exc_info=True) + # TODO: maybe we should call the callback with a disconnected event or do it on reconnect, + # so the callback has a chance to perform the logic to recover. await asyncio.sleep(1) finally: try: + # Restart subscription await self._driver.unsubscribe(channel_name) + channel = await self._driver.subscribe(channel_name) except Exception: pass @@ -90,8 +93,9 @@ async def subscribe(self, channel_name: str, rid: str, callback: Callable[[str], self._subscribers[channel_name][rid] = callback else: self._subscribers[channel_name] = {rid: callback} - task = asyncio.ensure_future(self.real_subscribe(channel_name)) - self._tasks[channel_name] = task + # Moved the subscribe command outside the future to ensure we are subscribed after returning + channel = await self._driver.subscribe(channel_name) + self._tasks[channel_name] = asyncio.ensure_future(self.real_subscribe(channel, channel_name)) async def unsubscribe(self, channel_name: str, req_id: str): if self._driver is None: diff --git a/guillotina/contrib/redis/driver.py b/guillotina/contrib/redis/driver.py index 334969fae..6e1134cc1 100644 --- a/guillotina/contrib/redis/driver.py +++ b/guillotina/contrib/redis/driver.py @@ -1,14 +1,14 @@ try: import aioredis - import aioredis.errors except ImportError: print("If you add guillotina.contrib.redis you need to add aioredis on your requirements") raise +from aioredis.client import PubSub from guillotina import app_settings from guillotina import metrics from guillotina.contrib.redis.exceptions import NoRedisConfigured -from typing import Any +from typing import Dict from typing import List from typing import Optional @@ -63,6 +63,7 @@ async def initialize(self, loop): while True: try: await self._connect() + assert await self._pool.ping() is True self.initialized = True break except Exception: # pragma: no cover @@ -71,18 +72,12 @@ async def initialize(self, loop): @backoff.on_exception(backoff.expo, (OSError,), max_time=30, max_tries=4) async def _connect(self): settings = app_settings["redis"] - with watch("create_pool"): - self._pool = await aioredis.create_pool( - (settings["host"], settings["port"]), **settings["pool"], loop=self._loop - ) - with watch("acquire_conn"): - self._conn = await self._pool.acquire() - self._pubsub_subscriptor = aioredis.Redis(self._conn) + self._conn_pool = aioredis.ConnectionPool.from_url(f"redis://{settings['host']}:{settings['port']}") + self._pool = aioredis.Redis(connection_pool=self._conn_pool) + self._pubsub_channels: Dict[str, PubSub] = {} async def finalize(self): - if self._pool is not None: - self._pool.close() - await self._pool.wait_closed() + await self._conn_pool.disconnect() self.initialized = False @property @@ -90,25 +85,27 @@ def pool(self): return self._pool async def info(self): - return await self._pool.execute(b"COMMAND", b"INFO", "get") + if self._pool is None: + raise NoRedisConfigured() + return await self._pool.info("get") # VALUE API async def set(self, key: str, data: str, *, expire: Optional[int] = None): if self._pool is None: raise NoRedisConfigured() - args: List[Any] = [] + kwargs = {} if expire is not None: - args[:] = [b"EX", expire] + kwargs["ex"] = expire with watch("set"): - ok = await self._pool.execute(b"SET", key, data, *args) - assert ok == b"OK", ok + ok = await self._pool.set(key, data, **kwargs) + assert ok is True, ok async def get(self, key: str) -> str: if self._pool is None: raise NoRedisConfigured() with watch("get") as w: - val = await self._pool.execute(b"GET", key) + val = await self._pool.get(key) if not val: w.labels["type"] = "get_miss" return val @@ -117,17 +114,17 @@ async def delete(self, key: str): if self._pool is None: raise NoRedisConfigured() with watch("delete"): - await self._pool.execute(b"DEL", key) + await self._pool.delete(key) async def expire(self, key: str, expire: int): if self._pool is None: raise NoRedisConfigured() - await self._pool.execute(b"EXPIRE", key, expire) + await self._pool.expire(key, expire) async def keys_startswith(self, key: str): if self._pool is None: raise NoRedisConfigured() - return await self._pool.execute(b"KEYS", f"{key}*") + return await self._pool.keys(f"{key}*") async def delete_all(self, keys: List[str]): if self._pool is None: @@ -135,55 +132,48 @@ async def delete_all(self, keys: List[str]): for key in keys: try: with watch("delete_many"): - await self._pool.execute(b"DEL", key) + await self._pool.delete(key) logger.debug("Deleted cache keys {}".format(keys)) except Exception: logger.warning("Error deleting cache keys {}".format(keys), exc_info=True) - async def flushall(self, *, async_op: Optional[bool] = False): + async def flushall(self, *, async_op: bool = False): if self._pool is None: raise NoRedisConfigured() - ops = [b"FLUSHDB"] - if async_op: - ops.append(b"ASYNC") with watch("flush"): - await self._pool.execute(*ops) + await self._pool.flushdb(asynchronous=async_op) # PUBSUB API async def publish(self, channel_name: str, data: str): if self._pool is None: raise NoRedisConfigured() + with watch("publish"): - await self._pool.execute(b"publish", channel_name, data) + await self._pool.publish(channel_name, data) async def unsubscribe(self, channel_name: str): - if self._pubsub_subscriptor is None: + if self._pool is None: raise NoRedisConfigured() + + p = self._pubsub_channels[channel_name] try: - await self._pubsub_subscriptor.unsubscribe(channel_name) - except aioredis.errors.ConnectionClosedError: - if self.initialized: - raise + await p.unsubscribe(channel_name) + finally: + await p.__aexit__(None, None, None) + del self._pubsub_channels[channel_name] async def subscribe(self, channel_name: str): - if self._pubsub_subscriptor is None: + if self._pool is None: raise NoRedisConfigured() - try: - (channel,) = await self._pubsub_subscriptor.subscribe(channel_name) - except aioredis.errors.ConnectionClosedError: # pragma: no cover - # closed in middle - try: - self._pool.close(self._conn) - except Exception: - pass - self._conn = await self._pool.acquire() - self._pubsub_subscriptor = aioredis.Redis(self._conn) - (channel,) = await self._pubsub_subscriptor.subscribe(channel_name) - return self._listener(channel) + p: PubSub = await self._pool.pubsub().__aenter__() + self._pubsub_channels[channel_name] = p + await p.subscribe(channel_name) + return self._listener(p) - async def _listener(self, channel: aioredis.Channel): - while await channel.wait_message(): - msg = await channel.get() - yield msg + async def _listener(self, p: PubSub): + while True: + message = await p.get_message(ignore_subscribe_messages=True, timeout=1) + if message is not None: + yield message["data"] diff --git a/guillotina/db/cache/base.py b/guillotina/db/cache/base.py index a18594040..c02929f4e 100644 --- a/guillotina/db/cache/base.py +++ b/guillotina/db/cache/base.py @@ -77,7 +77,7 @@ async def get(self, oid=None, container=None, id=None, variant=None): - tid: transaction id for ob - id """ - raise NotImplemented() + raise NotImplementedError async def set( self, value, keyset: List[Dict[str, Any]] = None, oid=None, container=None, id=None, variant=None @@ -85,16 +85,16 @@ async def set( """ Use params to build cache key """ - raise NotImplemented() + raise NotImplementedError async def clear(self): - raise NotImplemented() + raise NotImplementedError async def delete(self, key): - raise NotImplemented() + raise NotImplementedError async def delete_all(self, keys): - raise NotImplemented() + raise NotImplementedError async def store_object(self, obj, pickled): pass diff --git a/guillotina/db/factory.py b/guillotina/db/factory.py index 665ef873b..4e229a6e8 100644 --- a/guillotina/db/factory.py +++ b/guillotina/db/factory.py @@ -117,7 +117,7 @@ async def DummyFileDatabaseConfigurationFactory(key, dbconfig, loop=None): def _safe_db_name(name): - return "".join([l for l in name if l in string.digits + string.ascii_lowercase + "-_"]) + return "".join([l for l in name if l in string.digits + string.ascii_lowercase + "-_"]) # noqa: E741 @configure.adapter(for_=IApplication, provides=IDatabaseManager, name="postgresql") # noqa: N801 diff --git a/guillotina/db/storages/pg.py b/guillotina/db/storages/pg.py index 40e0e7f9e..2b0c41386 100644 --- a/guillotina/db/storages/pg.py +++ b/guillotina/db/storages/pg.py @@ -16,6 +16,7 @@ from guillotina.exceptions import ConflictIdOnContainer from guillotina.exceptions import TIDConflictError from guillotina.profile import profilable +from guillotina.task_vars import copy_context from zope.interface import implementer import asyncio @@ -384,7 +385,7 @@ class PGVacuum: def __init__(self, manager, loop): self._manager = manager self._loop = loop - self._queue = asyncio.Queue(loop=loop) + self._queue = asyncio.Queue() self._closed = False self._active = False self._sql = SQLStatements() @@ -403,7 +404,7 @@ async def _initialize(self): try: oid, table_name = await self._queue.get() self._active = True - await shield(self.vacuum(oid, table_name)) + await shield(copy_context(self.vacuum(oid, table_name))) except (concurrent.futures.CancelledError, RuntimeError): raise except Exception: @@ -555,7 +556,7 @@ async def initialize(self, loop=None, **kw): if self._autovacuum: self._vacuum = self._vacuum_class(self, loop) - self._vacuum_task = asyncio.Task(self._vacuum.initialize(), loop=loop) + self._vacuum_task = asyncio.Task(copy_context(self._vacuum.initialize()), loop=loop) async def restart(self, timeout=2): # needs to be used with lock diff --git a/guillotina/db/transaction.py b/guillotina/db/transaction.py index 5874f9b1d..de46188c7 100644 --- a/guillotina/db/transaction.py +++ b/guillotina/db/transaction.py @@ -34,6 +34,7 @@ import logging import sys import time +import warnings _EMPTY = "____" @@ -146,6 +147,9 @@ class Transaction: def __init__( self, manager, loop=None, read_only: bool = False, cache=None, strategy=None, ): + if loop is not None: + warnings.warn("Argument 'loop' is deprecated and ignored", DeprecationWarning) + # Transaction Manager self._manager = manager @@ -161,7 +165,7 @@ def __init__( # some databases need to lock during queries # this provides a lock for each transaction # which would correspond with one connection - self._lock = asyncio.Lock(loop=loop) + self._lock = asyncio.Lock() def initialize( self, read_only, cache=None, strategy=None, diff --git a/guillotina/db/transaction_manager.py b/guillotina/db/transaction_manager.py index ec6bef2db..54969acc8 100644 --- a/guillotina/db/transaction_manager.py +++ b/guillotina/db/transaction_manager.py @@ -12,6 +12,7 @@ from guillotina.exceptions import TIDConflictError from guillotina.exceptions import TransactionNotFound from guillotina.profile import profilable +from guillotina.task_vars import copy_context from guillotina.transactions import transaction from guillotina.utils import get_authenticated_user_id from zope.interface import implementer @@ -98,7 +99,7 @@ async def begin(self, read_only: bool = False) -> ITransaction: return txn async def commit(self, *, txn: typing.Optional[ITransaction] = None) -> None: - return await shield(self._commit(txn=txn)) + return await shield(copy_context(self._commit(txn=txn))) async def _commit(self, *, txn: typing.Optional[ITransaction] = None) -> None: """ Commit the last transaction @@ -153,7 +154,7 @@ async def _close_txn(self, txn: typing.Optional[ITransaction]): async def abort(self, *, txn: typing.Optional[ITransaction] = None) -> None: try: - return await shield(self._abort(txn=txn)) + return await shield(copy_context(self._abort(txn=txn))) except asyncio.CancelledError: pass diff --git a/guillotina/factory/content.py b/guillotina/factory/content.py index 290611192..477dde9f0 100644 --- a/guillotina/factory/content.py +++ b/guillotina/factory/content.py @@ -14,6 +14,7 @@ from guillotina.db.transaction_manager import TransactionManager from guillotina.interfaces import IApplication from guillotina.interfaces import IDatabase +from guillotina.task_vars import copy_context from guillotina.transactions import get_transaction from guillotina.utils import apply_coroutine from guillotina.utils import import_class @@ -69,7 +70,7 @@ def add_async_utility( kw["name"] = config["name"] provide_utility(utility_object, interface, **kw) if hasattr(utility_object, "initialize"): - func = lazy_apply(utility_object.initialize, app=self.app) + func = copy_context(lazy_apply(utility_object.initialize, app=self.app)) task = asyncio.ensure_future(notice_on_error(key, func), loop=loop or self._loop) self.add_async_task(key, task, config) diff --git a/guillotina/interfaces/security.py b/guillotina/interfaces/security.py index a81554af4..722478d90 100644 --- a/guillotina/interfaces/security.py +++ b/guillotina/interfaces/security.py @@ -64,7 +64,7 @@ def __str__(self): # register PermissionSettings to be symbolic constants by identity, # even when pickled and unpickled. copyreg.constructor(PermissionSetting) -copyreg.pickle(PermissionSetting, PermissionSetting.get_name, PermissionSetting) +copyreg.pickle(PermissionSetting, PermissionSetting.get_name, PermissionSetting) # type: ignore Allow = PermissionSetting("Allow", "Explicit allow setting for permissions") diff --git a/guillotina/metrics.py b/guillotina/metrics.py index 4fed1b591..865ed5f14 100644 --- a/guillotina/metrics.py +++ b/guillotina/metrics.py @@ -25,7 +25,7 @@ def __init__( *, counter: Optional[Counter] = None, histogram: Optional[Histogram] = None, - error_mappings: Dict[str, Type[Exception]] = None, + error_mappings: Dict[str, Type[BaseException]] = None, labels: Optional[Dict[str, str]] = None, ): self.counter = counter @@ -39,7 +39,7 @@ def __enter__(self): def __exit__( self, - exc_type: Optional[Type[Exception]], + exc_type: Optional[Type[BaseException]], exc_value: Optional[Exception], exc_traceback: Optional[traceback.StackSummary], ): diff --git a/guillotina/request.py b/guillotina/request.py index fc1df6cad..902e63293 100644 --- a/guillotina/request.py +++ b/guillotina/request.py @@ -508,7 +508,7 @@ def headers(self) -> "multidict.CIMultiDict[str]": @reify def cookies(self) -> dict: - cookie_jar = SimpleCookie(self.headers.get("cookie") or "") + cookie_jar = SimpleCookie(self.headers.get("cookie") or "") # type: ignore return {name: value.value for name, value in cookie_jar.items()} @property diff --git a/guillotina/task_vars.py b/guillotina/task_vars.py index 50ead784c..d6f219ba7 100644 --- a/guillotina/task_vars.py +++ b/guillotina/task_vars.py @@ -1,4 +1,3 @@ -from contextvars import ContextVar from guillotina.db.interfaces import ITransaction from guillotina.db.interfaces import ITransactionManager from guillotina.interfaces import IContainer @@ -8,17 +7,156 @@ from guillotina.interfaces import IRequest from guillotina.interfaces import ISecurityPolicy from typing import Dict +from typing import Generic from typing import Optional +from typing import TypeVar +import asyncio +import contextvars +import weakref -request: ContextVar[Optional[IRequest]] = ContextVar("g_request", default=None) -txn: ContextVar[Optional[ITransaction]] = ContextVar("g_txn", default=None) -tm: ContextVar[Optional[ITransactionManager]] = ContextVar("g_tm", default=None) -futures: ContextVar[Optional[Dict]] = ContextVar("g_futures", default=None) -authenticated_user: ContextVar[Optional[IPrincipal]] = ContextVar("g_authenticated_user", default=None) -security_policies: ContextVar[Optional[Dict[str, ISecurityPolicy]]] = ContextVar( + +# This global dictionary keeps all the contextvars for each task. +# When a task finishes and is destroyed, the context is destroyed as well +_context = weakref.WeakKeyDictionary() # type: ignore + + +class FakeTask: + """ + This class is necessary because we need an object to use as a key in the WeakKeyDictionary _context. + We can't use built-in objects because they don't have the `__weakref__` in the `__slots__` and without + this attribute, weakreaf.ref doesn't work + """ + + +_no_task_fallback = FakeTask() + + +def copy_context(coro): + """ + This function it's similar to contextvars.copy_context() but has a slightly different + signature and it's not called by default when a new task/future is created. + + To copy the context from the current task to a new one you need to call this + funcion explicitly, like this: + + async def worker(): + ... + + asyncio.create_task(copy_context(worker())) + + """ + try: + from_task = asyncio.current_task() + except RuntimeError: + assert _no_task_fallback is not None + from_task = _no_task_fallback + + if from_task in _context: + # The _context value type is a dict so we need to copy the dict to avoid + # sharing the same context value in different tasks + new_context = _context[from_task].copy() + else: + new_context = {} + + return _run_coro_with_ctx(coro, new_context) + + +async def _run_coro_with_ctx(coro, new_context): + task = asyncio.current_task() + _context[task] = new_context + return await coro + + +_T = TypeVar("_T") +_NO_DEFAULT = object() + + +class Token: + """ + Reimplementation of contextvars.Token + """ + + MISSING = contextvars.Token.MISSING + + def __init__(self, var, old_value) -> None: + self._var = var + self._old_value = old_value + + @property + def var(self): + return self._var + + @property + def old_value(self): + return self._old_value + + +class ShyContextVar(Generic[_T]): + """ + Reimplementation of contextvars.ContextVar but stores the values to the global `_context` + instead of storing it to the PyContext + """ + + def __init__(self, name: str, default=_NO_DEFAULT): + self._name = name + self._default = default + + @property + def name(self): + return self._name + + def get(self, default=_NO_DEFAULT): + ctx = self._get_ctx_data() + if self._name in ctx: + return ctx[self._name] + elif default != _NO_DEFAULT: + return default + elif self._default != _NO_DEFAULT: + return self._default + else: + raise LookupError(self) + + def set(self, value) -> Token: + data = self._get_ctx_data() + name = self._name + if name in data: + t = Token(self, data[name]) + else: + t = Token(self, Token.MISSING) + data[self._name] = value + return t + + def reset(self, token): + if token.old_value == Token.MISSING: + ctx = self._get_ctx_data() + if ctx and self._name in ctx: + del ctx[self._name] + else: + self.set(token.old_value) + + def _get_ctx_data(self): + try: + task = asyncio.current_task() + except RuntimeError: + task = _no_task_fallback + try: + data = _context[task] + except KeyError: + # Initialize _context value for this task + data = {} + _context[task] = data + return data + + +request: ShyContextVar[Optional[IRequest]] = ShyContextVar("g_request", default=None) +txn: ShyContextVar[Optional[ITransaction]] = ShyContextVar("g_txn", default=None) +tm: ShyContextVar[Optional[ITransactionManager]] = ShyContextVar("g_tm", default=None) +futures: ShyContextVar[Optional[Dict]] = ShyContextVar("g_futures", default=None) +authenticated_user: ShyContextVar[Optional[IPrincipal]] = ShyContextVar("g_authenticated_user", default=None) +security_policies: ShyContextVar[Optional[Dict[str, ISecurityPolicy]]] = ShyContextVar( "g_security_policy", default=None ) -container: ContextVar[Optional[IContainer]] = ContextVar("g_container", default=None) -registry: ContextVar[Optional[IRegistry]] = ContextVar("g_container", default=None) -db: ContextVar[Optional[IDatabase]] = ContextVar("g_database", default=None) +container: ShyContextVar[Optional[IContainer]] = ShyContextVar("g_container", default=None) +registry: ShyContextVar[Optional[IRegistry]] = ShyContextVar("g_registry", default=None) +db: ShyContextVar[Optional[IDatabase]] = ShyContextVar("g_database", default=None) diff --git a/guillotina/tests/fixtures.py b/guillotina/tests/fixtures.py index 5743ab3bc..894574d15 100644 --- a/guillotina/tests/fixtures.py +++ b/guillotina/tests/fixtures.py @@ -13,6 +13,7 @@ from guillotina.interfaces import IDatabase from guillotina.tests import mocks from guillotina.tests.utils import ContainerRequesterAsyncContextManager +from guillotina.tests.utils import copy_global_ctx from guillotina.tests.utils import get_mocked_request from guillotina.tests.utils import login from guillotina.tests.utils import logout @@ -26,7 +27,6 @@ import asyncio import json import os -import prometheus_client.registry import pytest @@ -386,6 +386,7 @@ def clear_task_vars(): @pytest.fixture(scope="function") async def dummy_guillotina(event_loop, request): globalregistry.reset() + task_vars._no_task_fallback = task_vars.FakeTask() app = make_app(settings=get_dummy_settings(request.node), loop=event_loop) async with TestClient(app): yield app @@ -399,13 +400,13 @@ def __init__(self, dummy_request, loop): self.loop = loop async def __aenter__(self): - task = asyncio.Task.current_task(loop=self.loop) + task = asyncio.current_task(loop=self.loop) if task is not None: task.request = self.request return self.request async def __aexit__(self, exc_type, exc, tb): - task = asyncio.Task.current_task(loop=self.loop) + task = asyncio.current_task(loop=self.loop) del task.request @@ -429,6 +430,10 @@ def __init__(self, request): self.txn = None async def __aenter__(self): + # This is a hack to copy contextvars defined in fixture dummy_request + # (oustide event loop) to this asyncio task + copy_global_ctx() + tm = get_tm() self.txn = await tm.begin() self.root = await tm.get_root() @@ -484,6 +489,7 @@ async def _clear_dbs(root): @pytest.fixture(scope="function") async def app(event_loop, db, request): globalregistry.reset() + task_vars._no_task_fallback = task_vars.FakeTask() settings = get_db_settings(request.node) app = make_app(settings=settings, loop=event_loop) @@ -516,6 +522,7 @@ async def app(event_loop, db, request): @pytest.fixture(scope="function") async def app_client(event_loop, db, request): globalregistry.reset() + task_vars._no_task_fallback = task_vars.FakeTask() app = make_app(settings=get_db_settings(request.node), loop=event_loop) async with TestClient(app, timeout=30) as client: await _clear_dbs(app.app.root) @@ -651,6 +658,8 @@ async def dbusers_requester(guillotina): @pytest.fixture(scope="function") async def metrics_registry(): + import prometheus_client.registry + for collector in prometheus_client.registry.REGISTRY._names_to_collectors.values(): if not hasattr(collector, "_metrics"): continue diff --git a/guillotina/tests/memcached/test_cache.py b/guillotina/tests/memcached/test_cache.py index 1607ee41d..543a716e9 100644 --- a/guillotina/tests/memcached/test_cache.py +++ b/guillotina/tests/memcached/test_cache.py @@ -1,5 +1,8 @@ +try: + from guillotina.contrib.memcached.driver import MemcachedDriver +except ImportError: + MemcachedDriver = None # type: ignore from guillotina.component import get_utility -from guillotina.contrib.memcached.driver import MemcachedDriver from guillotina.interfaces import ICacheUtility import pytest @@ -14,6 +17,7 @@ } +@pytest.mark.skipif(MemcachedDriver is None, reason="emcache not installed") @pytest.mark.app_settings(MEMCACHED_SETTINGS) async def test_cache_uses_memcached_driver_when_configured(memcached_container, guillotina_main): cache = get_utility(ICacheUtility) diff --git a/guillotina/tests/memcached/test_memcached_driver.py b/guillotina/tests/memcached/test_memcached_driver.py index fbb6a06c4..d7d5a1250 100644 --- a/guillotina/tests/memcached/test_memcached_driver.py +++ b/guillotina/tests/memcached/test_memcached_driver.py @@ -1,11 +1,15 @@ -from guillotina.contrib.memcached.driver import MemcachedDriver -from guillotina.contrib.memcached.driver import safe_key -from guillotina.contrib.memcached.driver import update_connection_pool_metrics +try: + import emcache + from guillotina.contrib.memcached.driver import MemcachedDriver + from guillotina.contrib.memcached.driver import safe_key + from guillotina.contrib.memcached.driver import update_connection_pool_metrics +except ModuleNotFoundError: + emcache = None + from guillotina.utils import resolve_dotted_name from unittest import mock import asyncio -import emcache import pytest @@ -32,6 +36,7 @@ def mocked_create_client(): yield create_client +@pytest.mark.skipif(emcache is None, reason="emcache not installed") async def test_create_client_returns_emcache_client(memcached_container, guillotina_main): driver = MemcachedDriver() assert driver.client is None @@ -41,6 +46,7 @@ async def test_create_client_returns_emcache_client(memcached_container, guillot assert isinstance(client, emcache.Client) +@pytest.mark.skipif(emcache is None, reason="emcache not installed") async def test_client_is_initialized_with_configured_hosts(mocked_create_client): settings = {"hosts": MOCK_HOSTS} driver = MemcachedDriver() @@ -48,6 +54,7 @@ async def test_client_is_initialized_with_configured_hosts(mocked_create_client) assert len(mocked_create_client.call_args[0][0]) == 1 +@pytest.mark.skipif(emcache is None, reason="emcache not installed") async def test_create_client_ignores_invalid_params(mocked_create_client): settings = {"hosts": MOCK_HOSTS} driver = MemcachedDriver() @@ -55,6 +62,7 @@ async def test_create_client_ignores_invalid_params(mocked_create_client): assert mocked_create_client.call_args[1] == {} +@pytest.mark.skipif(emcache is None, reason="emcache not installed") @pytest.mark.parametrize( "param,values", [ @@ -74,6 +82,7 @@ async def test_create_client_sets_configured_params(mocked_create_client, param, assert mocked_create_client.call_args[1][param] == value +@pytest.mark.skipif(emcache is None, reason="emcache not installed") @pytest.mark.app_settings(MEMCACHED_SETTINGS) async def test_memcached_ops(memcached_container, guillotina_main, dont_probe_metrics): driver = await resolve_dotted_name("guillotina.contrib.memcached").get_driver() @@ -114,6 +123,7 @@ async def test_memcached_ops(memcached_container, guillotina_main, dont_probe_me unsafe_keys = ["a" * 255, "foo bar", b"\x130".decode()] +@pytest.mark.skipif(emcache is None, reason="emcache not installed") @pytest.mark.app_settings(MEMCACHED_SETTINGS) @pytest.mark.parametrize("unsafe_key", unsafe_keys) async def test_memcached_ops_are_safe_key( @@ -126,6 +136,7 @@ async def test_memcached_ops_are_safe_key( await driver.delete_all([unsafe_key]) +@pytest.mark.skipif(emcache is None, reason="emcache not installed") @pytest.mark.parametrize("unsafe_key", unsafe_keys) async def test_safe_key(unsafe_key): key = safe_key(unsafe_key) @@ -136,6 +147,7 @@ async def test_safe_key(unsafe_key): assert char <= 126 +@pytest.mark.skipif(emcache is None, reason="emcache not installed") async def test_delete_all(): with mock.patch("guillotina.contrib.memcached.driver.watch") as watch_mocked: with mock.patch("guillotina.contrib.memcached.driver.MEMCACHED_OPS_DELETE_ALL_NUM_KEYS") as all_keys: @@ -149,6 +161,7 @@ async def test_delete_all(): ) +@pytest.mark.skipif(emcache is None, reason="emcache not installed") async def test_delete_all_empty_keys(): with mock.patch("guillotina.contrib.memcached.driver.watch") as watch_mocked: with mock.patch("guillotina.contrib.memcached.driver.MEMCACHED_OPS_DELETE_ALL_NUM_KEYS") as all_keys: @@ -159,6 +172,7 @@ async def test_delete_all_empty_keys(): watch_mocked.assert_not_called() +@pytest.mark.skipif(emcache is None, reason="emcache not installed") class TestUpdateConnectionPoolMetrics: @pytest.fixture def avg(self): diff --git a/guillotina/tests/memcached/test_metrics.py b/guillotina/tests/memcached/test_metrics.py index dd1729954..d5cbe9150 100644 --- a/guillotina/tests/memcached/test_metrics.py +++ b/guillotina/tests/memcached/test_metrics.py @@ -1,5 +1,9 @@ +try: + from guillotina.contrib.memcached.driver import MemcachedDriver +except ModuleNotFoundError: + MemcachedDriver = None # type: ignore + from asyncmock import AsyncMock -from guillotina.contrib.memcached.driver import MemcachedDriver import pytest @@ -7,6 +11,7 @@ pytestmark = pytest.mark.asyncio +@pytest.mark.skipif(MemcachedDriver is None, reason="emcache not installed") class TestMemcachedMetrics: async def test_connect_metric(self, metrics_registry, event_loop): driver = MemcachedDriver() diff --git a/guillotina/tests/test_auth.py b/guillotina/tests/test_auth.py index e8589f6ac..3893918b8 100644 --- a/guillotina/tests/test_auth.py +++ b/guillotina/tests/test_auth.py @@ -18,7 +18,7 @@ async def test_jwt_auth(container_requester): {"exp": datetime.utcnow() + timedelta(seconds=60), "id": ROOT_USER_ID}, app_settings["jwt"]["secret"], algorithm=app_settings["jwt"]["algorithm"], - ).decode("utf-8") + ) response, status = await requester( "GET", "/db/guillotina/@addons", token=jwt_token, auth_type="Bearer" @@ -43,7 +43,7 @@ async def test_cookie_auth(container_requester): {"exp": datetime.utcnow() + timedelta(seconds=60), "id": ROOT_USER_ID}, app_settings["jwt"]["secret"], algorithm=app_settings["jwt"]["algorithm"], - ).decode("utf-8") + ) response, status = await requester( "GET", "/db/guillotina/@addons", authenticated=False, cookies={"auth_token": jwt_token} diff --git a/guillotina/tests/test_metrics.py b/guillotina/tests/test_metrics.py index fdf289a8d..1533cecf2 100644 --- a/guillotina/tests/test_metrics.py +++ b/guillotina/tests/test_metrics.py @@ -23,7 +23,7 @@ class TestRedisMetrics: async def test_set_redis_metric(self, metrics_registry): driver = RedisDriver() driver._pool = AsyncMock() - driver._pool.execute.return_value = b"OK" + driver._pool.set.return_value = True await driver.set("foo", "bar") assert ( @@ -59,7 +59,7 @@ async def test_get_redis_metric(self, metrics_registry): async def test_get_miss_redis_metric(self, metrics_registry): driver = RedisDriver() driver._pool = AsyncMock() - driver._pool.execute.return_value = None + driver._pool.get.return_value = None await driver.get("foo") assert ( metrics_registry.get_sample_value( diff --git a/guillotina/tests/test_serialize.py b/guillotina/tests/test_serialize.py index 3e6f9e61b..2ecca57e8 100644 --- a/guillotina/tests/test_serialize.py +++ b/guillotina/tests/test_serialize.py @@ -16,6 +16,7 @@ from guillotina.json.serialize_value import json_compatible from guillotina.json.utils import validate_invariants from guillotina.schema.exceptions import WrongType +from guillotina.tests.utils import copy_global_ctx from guillotina.tests.utils import create_content from guillotina.tests.utils import login from zope.interface import Interface @@ -30,6 +31,7 @@ async def test_serialize_resource(dummy_request, mock_txn): + copy_global_ctx() content = create_content() content.creation_date = datetime(2020, 1, 1, 10, 10, 10, tzinfo=tzutc()) serializer = get_multi_adapter((content, dummy_request), IResourceSerializeToJson) @@ -39,6 +41,7 @@ async def test_serialize_resource(dummy_request, mock_txn): async def test_serialize_resource_omit_behavior(dummy_request, mock_txn): + copy_global_ctx() content = create_content() serializer = get_multi_adapter((content, dummy_request), IResourceSerializeToJson) result = await serializer(omit=["guillotina.behaviors.dublincore.IDublinCore"]) @@ -46,6 +49,7 @@ async def test_serialize_resource_omit_behavior(dummy_request, mock_txn): async def test_serialize_resource_omit_field(dummy_request, mock_txn): + copy_global_ctx() content = create_content() serializer = get_multi_adapter((content, dummy_request), IResourceSerializeToJson) result = await serializer(omit=["guillotina.behaviors.dublincore.IDublinCore.creators"]) @@ -53,6 +57,7 @@ async def test_serialize_resource_omit_field(dummy_request, mock_txn): async def test_serialize_resource_include_field(dummy_request, mock_txn): + copy_global_ctx() from guillotina.test_package import FileContent obj = create_content(FileContent, type_name="File") @@ -65,6 +70,7 @@ async def test_serialize_resource_include_field(dummy_request, mock_txn): async def test_serialize_omit_main_interface_field(dummy_request, mock_txn): + copy_global_ctx() from guillotina.test_package import FileContent obj = create_content(FileContent, type_name="File") @@ -77,6 +83,7 @@ async def test_serialize_omit_main_interface_field(dummy_request, mock_txn): async def test_serialize_cloud_file(dummy_request, mock_txn): + copy_global_ctx() from guillotina.test_package import FileContent, IFileContent from guillotina.interfaces import IFileManager @@ -100,6 +107,7 @@ async def _data(): async def test_deserialize_cloud_file(dummy_request, mock_txn): + copy_global_ctx() from guillotina.test_package import IFileContent, FileContent obj = create_content(FileContent) @@ -212,6 +220,7 @@ async def test_deserialize_float_convert_int(dummy_guillotina): async def test_sub_field_float_convert_int(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -273,6 +282,7 @@ async def test_check_permission_deserialize_content(dummy_request): async def test_patch_list_field_normal_patch(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -281,6 +291,7 @@ async def test_patch_list_field_normal_patch(dummy_request, mock_txn): async def test_patch_list_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -326,6 +337,7 @@ async def test_patch_list_field(dummy_request, mock_txn): async def test_patch_tuple_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -371,6 +383,7 @@ async def test_patch_tuple_field(dummy_request, mock_txn): async def test_patch_list_field_invalid_type(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -383,6 +396,7 @@ async def test_patch_list_field_invalid_type(dummy_request, mock_txn): async def test_patch_dict_field_normal_patch(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -391,6 +405,7 @@ async def test_patch_dict_field_normal_patch(dummy_request, mock_txn): async def test_patch_dict_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -419,6 +434,7 @@ async def test_patch_dict_field(dummy_request, mock_txn): async def test_patch_dict_field_invalid_type(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -433,6 +449,7 @@ async def test_patch_dict_field_invalid_type(dummy_request, mock_txn): async def test_patch_int_field_normal_path(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -441,6 +458,7 @@ async def test_patch_int_field_normal_path(dummy_request, mock_txn): async def test_patch_int_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -496,6 +514,7 @@ async def test_patch_int_field_invalid_type(dummy_request): async def test_bucket_list_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -550,6 +569,7 @@ async def test_bucket_list_field(dummy_request, mock_txn): async def test_bucket_list_field_clear(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -620,6 +640,7 @@ async def test_default_value_deserialize(dummy_request): async def test_nested_patch_deserialize(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -694,6 +715,7 @@ async def test_nested_patch_deserialize(dummy_request, mock_txn): async def test_object_deserialize(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -703,6 +725,7 @@ async def test_object_deserialize(dummy_request, mock_txn): async def test_dates_bucket_list_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -728,6 +751,7 @@ async def test_dates_bucket_list_field(dummy_request, mock_txn): async def test_patchfield_notdefined_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -791,6 +815,7 @@ async def test_patchfield_notdefined_field(dummy_request, mock_txn): async def test_delete_by_value_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -812,6 +837,7 @@ async def test_delete_by_value_field(dummy_request, mock_txn): async def test_bucket_dict_field(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -878,6 +904,7 @@ async def test_bucket_dict_field(dummy_request, mock_txn): async def test_unhandled_exceptions_in_bucket_dict_field_do_not_write_to_object(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -890,6 +917,7 @@ async def test_unhandled_exceptions_in_bucket_dict_field_do_not_write_to_object( async def test_bucket_dict_field_splitting(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -903,6 +931,7 @@ async def test_bucket_dict_field_splitting(dummy_request, mock_txn): ) assert len(mock_txn.added) == 1 # should all be in one bucket mock_txn.added.clear() + # now test inserting one that will split the buckets and insert into new bucket. await deserializer.set_schema( ITestSchema, @@ -914,6 +943,7 @@ async def test_bucket_dict_field_splitting(dummy_request, mock_txn): async def test_bucket_dict_field_clear(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -956,6 +986,7 @@ def validate_not_foo(field, value): async def test_invariant_error(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -973,6 +1004,7 @@ async def test_invariant_error(dummy_request, mock_txn): async def test_constraint_error(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -986,6 +1018,7 @@ async def test_constraint_error(dummy_request, mock_txn): async def test_validator_error(dummy_request, mock_txn): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) @@ -1053,6 +1086,7 @@ async def test_patch_dict_update_max_ops(dummy_guillotina): async def test_bucket_list_invalid_op(dummy_guillotina, mock_txn): + copy_global_ctx() field = fields.BucketListField(bucket_len=10, required=False, value_type=schema.Text()) content = create_content() with pytest.raises(ValueDeserializationError): @@ -1069,6 +1103,7 @@ async def test_patch_dict_validation(dummy_guillotina): async def test_bucket_list_max_ops(dummy_guillotina, mock_txn): + copy_global_ctx() field = fields.BucketListField(bucket_len=10, required=False, max_ops=1, value_type=schema.Text()) field.__name__ = "foobar" content = create_content() @@ -1078,6 +1113,7 @@ async def test_bucket_list_max_ops(dummy_guillotina, mock_txn): async def test_bucket_dict_max_ops(dummy_guillotina, mock_txn): + copy_global_ctx() field = fields.BucketDictField(bucket_len=10, required=False, max_ops=1, value_type=schema.Text()) field.__name__ = "foobar" content = create_content() @@ -1111,6 +1147,7 @@ async def test_async_invariant(): async def test_async_invariant_deserializer(dummy_guillotina, mock_txn, dummy_request): + copy_global_ctx() login() content = create_content() deserializer = get_multi_adapter((content, dummy_request), IResourceDeserializeFromJson) diff --git a/guillotina/tests/test_server.py b/guillotina/tests/test_server.py index eb5087e20..df9f6dbeb 100644 --- a/guillotina/tests/test_server.py +++ b/guillotina/tests/test_server.py @@ -29,7 +29,7 @@ async def test_trns_retries_with_app(container_requester): @pytest.mark.asyncio async def test_async_util_started_and_stopped(dummy_guillotina): util = get_utility(ITestAsyncUtility) - util.state == "init" + assert util.state == "init" config_utility = { "provides": "guillotina.test_package.ITestAsyncUtility", diff --git a/guillotina/tests/test_transactions.py b/guillotina/tests/test_transactions.py index 0ce705cde..0def3c652 100644 --- a/guillotina/tests/test_transactions.py +++ b/guillotina/tests/test_transactions.py @@ -17,16 +17,16 @@ pytestmark = pytest.mark.asyncio -async def test_no_tid_created_for_reads(dummy_request, event_loop): +async def test_no_tid_created_for_reads(dummy_request): tm = mocks.MockTransactionManager() - trns = Transaction(tm, loop=event_loop, read_only=True) + trns = Transaction(tm, read_only=True) await trns.tpc_begin() assert trns._tid is None -async def test_tid_created_for_writes(dummy_request, event_loop): +async def test_tid_created_for_writes(dummy_request): tm = mocks.MockTransactionManager() - trns = Transaction(tm, loop=event_loop) + trns = Transaction(tm) await trns.tpc_begin() assert trns._tid == 1 diff --git a/guillotina/tests/utils.py b/guillotina/tests/utils.py index 75315765c..c77638b87 100644 --- a/guillotina/tests/utils.py +++ b/guillotina/tests/utils.py @@ -171,3 +171,9 @@ def make_mocked_request( return Request( "http", method, path, query_string, raw_hdrs, client_max_size=client_max_size, receive=q.get ) + + +def copy_global_ctx(): + task = asyncio.current_task() + global_ctx = task_vars._context[task_vars._no_task_fallback] + task_vars._context[task] = global_ctx.copy() diff --git a/guillotina/traversal.py b/guillotina/traversal.py index c5711077c..23942ee28 100644 --- a/guillotina/traversal.py +++ b/guillotina/traversal.py @@ -84,7 +84,7 @@ async def traverse( if context is None: return parent, path else: - context = parent[path[0]] + context = parent[path[0]] # type: ignore except (TypeError, KeyError, AttributeError): return parent, path @@ -104,7 +104,10 @@ async def traverse( # there is an existing registry object set on task_vars task_vars.registry.set(None) registry = await get_registry(context) - layers = registry.get(ACTIVE_LAYERS_KEY, []) + if registry: + layers = registry.get(ACTIVE_LAYERS_KEY, []) + else: + layers = [] for layer in layers: try: alsoProvides(request, import_class(layer)) diff --git a/guillotina/utils/content.py b/guillotina/utils/content.py index 8f76c2c35..cf2a2cd47 100644 --- a/guillotina/utils/content.py +++ b/guillotina/utils/content.py @@ -72,7 +72,7 @@ def valid_id(_id) -> bool: # can't start with _ or be path explorers if _id in (None, ".", "..") or _id[0] in ("_", "@"): return False - return _id == "".join([l for l in _id if l in app_settings["valid_id_characters"]]) + return _id == "".join([l for l in _id if l in app_settings["valid_id_characters"]]) # noqa: E741 async def get_containers(): diff --git a/guillotina/utils/execute.py b/guillotina/utils/execute.py index b497ce21c..dea25dd3a 100644 --- a/guillotina/utils/execute.py +++ b/guillotina/utils/execute.py @@ -5,6 +5,7 @@ from guillotina.interfaces import IAsyncJobPool from guillotina.interfaces import IQueueUtility from guillotina.profile import profilable +from guillotina.task_vars import copy_context from guillotina.transactions import get_transaction from typing import Any from typing import Callable @@ -201,7 +202,7 @@ def execute_futures(scope: str = "", futures=None, task=None) -> Optional[asynci fut = fut_data["fut"] if not asyncio.iscoroutine(fut): fut = fut(*fut_data.get("args") or [], **fut_data.get("kwargs") or {}) - found.append(fut) + found.append(copy_context(fut)) futures[scope] = {} if len(found) > 0: task = asyncio.ensure_future(asyncio.gather(*found)) diff --git a/guillotina/utils/misc.py b/guillotina/utils/misc.py index 8eafb963a..0d7b35ab2 100644 --- a/guillotina/utils/misc.py +++ b/guillotina/utils/misc.py @@ -1,4 +1,4 @@ -from collections import MutableMapping +from collections.abc import MutableMapping from functools import partial from guillotina import glogging from guillotina import task_vars diff --git a/requirements.txt b/requirements.txt index b55e9ea11..b809e7493 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -Cython==0.29.13 -asyncpg==0.21.0 -cffi==1.12.2 +Cython==0.29.24 +asyncpg==0.24.0 +cffi==1.14.6 chardet==3.0.4 jsonschema==2.6.0 multidict==4.5.2 -pycparser==2.18 +pycparser==2.20 pycryptodome==3.6.6 -PyJWT==1.6.0 -python-dateutil==2.6.1 +PyJWT~=2.1.0 +python-dateutil==2.8.2 PyYaml>=5.1 six==1.11.0 orjson>=3,<4 @@ -16,4 +16,13 @@ uvicorn==0.13.1 argon2-cffi==18.3.0 backoff==1.10.0 prometheus-client==0.8.0 -typing_extensions==3.7.4.3 \ No newline at end of file +typing_extensions==3.7.4.3 + +types-chardet==0.1.5 +types-docutils==0.17.0 +types-orjson==3.6.0 +types-python-dateutil==0.1.6 +types-pytz==2021.1.2 +types-PyYAML==5.4.6 +types-setuptools==57.0.2 +types-toml==0.1.5 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index aaede9586..68e99f2cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ skip_glob=*.pyi [flake8] max_line_length = 120 no-accept-encodings = True +exclude = guillotina/cookiecutter ignore = E302 W391 @@ -27,11 +28,13 @@ ignore = W503 E203 BLK100 + F541 [mypy] namespace_packages=True plugins=mypy_zope:plugin mypy_path=stubs +exclude = guillotina/cookiecutter [mypy-pytest] ignore_missing_imports = True @@ -52,4 +55,16 @@ ignore_missing_imports = True ignore_missing_imports = True [mypy-asyncmock.*] -ignore_missing_imports = True \ No newline at end of file +ignore_missing_imports = True + +[mypy-PIL.*] +ignore_missing_imports = True + +[mypy-emcache.*] +ignore_missing_imports = True + +[mypy-jwt.*] +ignore_missing_imports = True + +[mypy-jinja2.*] +ignore_missing_imports = True diff --git a/setup.py b/setup.py index 687cbbae0..d18659576 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,8 @@ "Topic :: Internet :: WWW/HTTP", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Libraries :: Python Modules", ], url="https://github.com/plone/guillotina", @@ -63,7 +65,7 @@ "setuptools", "orjson>=3,<4", "zope.interface", - "pyjwt<2.0.0", + "pyjwt", "asyncpg", "cffi", "PyYAML>=5.1", @@ -76,7 +78,7 @@ ], extras_require={ "test": [ - "pytest>=3.8.0,<6.1.0", + "pytest>=3.8.0,<6.3.0", "docker", "backoff", "psycopg2-binary", @@ -84,7 +86,7 @@ "pytest-cov", "coverage>=4.0.3", "pytest-docker-fixtures", - "pytest-rerunfailures<=9.0", + "pytest-rerunfailures<=10.1", "async-asgi-testclient<2.0.0", "openapi-spec-validator==0.2.9", "aiohttp>=3.0.0,<4.0.0", @@ -104,7 +106,7 @@ 'aiohttp>=3.0.0,<3.6.0;python_version<"3.8"', 'aiohttp>=3.6.0,<4.0.0;python_version>="3.8"', ], - "redis": ['aioredis==1.3.1'], + "redis": ['aioredis==2.0.0'], "mailer": ["html2text>=2018.1.9", "aiosmtplib>=1.0.6"], "memcached": ["emcache"], "validation": ["pytz==2020.1"],