From c1986242df3bbad0c12b15bef7b60335539ba237 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 10 Oct 2023 16:29:06 +0200 Subject: [PATCH] Implement taskiq serializer --- poetry.lock | 114 ++++++++++++---------- pyproject.toml | 1 + taskiq/abc/broker.py | 20 +++- taskiq/abc/serializer.py | 24 +++++ taskiq/compat.py | 18 ++++ taskiq/formatters/json_formatter.py | 2 +- taskiq/formatters/proxy_formatter.py | 38 ++++++++ taskiq/message.py | 4 +- taskiq/serializers/__init__.py | 1 + taskiq/serializers/json_serializer.py | 26 +++++ tests/formatters/test_json_formatter.py | 45 +++++++++ tests/formatters/test_proxy_formatter.py | 47 +++++++++ tests/serializers/test_json_serializer.py | 23 +++++ 13 files changed, 306 insertions(+), 57 deletions(-) create mode 100644 taskiq/abc/serializer.py create mode 100644 taskiq/formatters/proxy_formatter.py create mode 100644 taskiq/serializers/__init__.py create mode 100644 taskiq/serializers/json_serializer.py create mode 100644 tests/formatters/test_json_formatter.py create mode 100644 tests/formatters/test_proxy_formatter.py create mode 100644 tests/serializers/test_json_serializer.py diff --git a/poetry.lock b/poetry.lock index 5a805d2..422d2d7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "annotated-types" -version = "0.5.0" +version = "0.6.0" description = "Reusable constraint types to use with typing.Annotated" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "annotated_types-0.5.0-py3-none-any.whl", hash = "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd"}, - {file = "annotated_types-0.5.0.tar.gz", hash = "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802"}, + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, ] [package.dependencies] @@ -356,12 +356,12 @@ python-dateutil = ">=2.7" [[package]] name = "gitignore-parser" -version = "0.1.6" +version = "0.1.9" description = "A spec-compliant gitignore parser for Python 3.5+" optional = true python-versions = "*" files = [ - {file = "gitignore_parser-0.1.6.tar.gz", hash = "sha256:8962420f7abb02cc9bb17461b37504eaa2342775408de6e5375f9c8a6c662fa7"}, + {file = "gitignore_parser-0.1.9.tar.gz", hash = "sha256:270cb8cd09de410b8805c5f4183fd404c28f910dcbb94e1efc08226144fdff7d"}, ] [[package]] @@ -377,13 +377,13 @@ files = [ [[package]] name = "identify" -version = "2.5.29" +version = "2.5.30" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.29-py2.py3-none-any.whl", hash = "sha256:24437fbf6f4d3fe6efd0eb9d67e24dd9106db99af5ceb27996a5f7895f24bf1b"}, - {file = "identify-2.5.29.tar.gz", hash = "sha256:d43d52b86b15918c137e3a74fff5224f60385cd0e9c38e99d07c257f02f151a5"}, + {file = "identify-2.5.30-py2.py3-none-any.whl", hash = "sha256:afe67f26ae29bab007ec21b03d4114f41316ab9dd15aa8736a167481e108da54"}, + {file = "identify-2.5.30.tar.gz", hash = "sha256:f302a4256a15c849b91cfcdcec052a8ce914634b2f77ae87dad29cd749f2d88d"}, ] [package.extras] @@ -448,38 +448,38 @@ test = ["pytest (<5.4)", "pytest-cov"] [[package]] name = "mypy" -version = "1.5.1" +version = "1.6.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f33592ddf9655a4894aef22d134de7393e95fcbdc2d15c1ab65828eee5c66c70"}, - {file = "mypy-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:258b22210a4a258ccd077426c7a181d789d1121aca6db73a83f79372f5569ae0"}, - {file = "mypy-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9ec1f695f0c25986e6f7f8778e5ce61659063268836a38c951200c57479cc12"}, - {file = "mypy-1.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:abed92d9c8f08643c7d831300b739562b0a6c9fcb028d211134fc9ab20ccad5d"}, - {file = "mypy-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a156e6390944c265eb56afa67c74c0636f10283429171018446b732f1a05af25"}, - {file = "mypy-1.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6ac9c21bfe7bc9f7f1b6fae441746e6a106e48fc9de530dea29e8cd37a2c0cc4"}, - {file = "mypy-1.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51cb1323064b1099e177098cb939eab2da42fea5d818d40113957ec954fc85f4"}, - {file = "mypy-1.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:596fae69f2bfcb7305808c75c00f81fe2829b6236eadda536f00610ac5ec2243"}, - {file = "mypy-1.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:32cb59609b0534f0bd67faebb6e022fe534bdb0e2ecab4290d683d248be1b275"}, - {file = "mypy-1.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:159aa9acb16086b79bbb0016145034a1a05360626046a929f84579ce1666b315"}, - {file = "mypy-1.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f6b0e77db9ff4fda74de7df13f30016a0a663928d669c9f2c057048ba44f09bb"}, - {file = "mypy-1.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26f71b535dfc158a71264e6dc805a9f8d2e60b67215ca0bfa26e2e1aa4d4d373"}, - {file = "mypy-1.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fc3a600f749b1008cc75e02b6fb3d4db8dbcca2d733030fe7a3b3502902f161"}, - {file = "mypy-1.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:26fb32e4d4afa205b24bf645eddfbb36a1e17e995c5c99d6d00edb24b693406a"}, - {file = "mypy-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:82cb6193de9bbb3844bab4c7cf80e6227d5225cc7625b068a06d005d861ad5f1"}, - {file = "mypy-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4a465ea2ca12804d5b34bb056be3a29dc47aea5973b892d0417c6a10a40b2d65"}, - {file = "mypy-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9fece120dbb041771a63eb95e4896791386fe287fefb2837258925b8326d6160"}, - {file = "mypy-1.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d28ddc3e3dfeab553e743e532fb95b4e6afad51d4706dd22f28e1e5e664828d2"}, - {file = "mypy-1.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:57b10c56016adce71fba6bc6e9fd45d8083f74361f629390c556738565af8eeb"}, - {file = "mypy-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:ff0cedc84184115202475bbb46dd99f8dcb87fe24d5d0ddfc0fe6b8575c88d2f"}, - {file = "mypy-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8f772942d372c8cbac575be99f9cc9d9fb3bd95c8bc2de6c01411e2c84ebca8a"}, - {file = "mypy-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5d627124700b92b6bbaa99f27cbe615c8ea7b3402960f6372ea7d65faf376c14"}, - {file = "mypy-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:361da43c4f5a96173220eb53340ace68cda81845cd88218f8862dfb0adc8cddb"}, - {file = "mypy-1.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:330857f9507c24de5c5724235e66858f8364a0693894342485e543f5b07c8693"}, - {file = "mypy-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:c543214ffdd422623e9fedd0869166c2f16affe4ba37463975043ef7d2ea8770"}, - {file = "mypy-1.5.1-py3-none-any.whl", hash = "sha256:f757063a83970d67c444f6e01d9550a7402322af3557ce7630d3c957386fa8f5"}, - {file = "mypy-1.5.1.tar.gz", hash = "sha256:b031b9601f1060bf1281feab89697324726ba0c0bae9d7cd7ab4b690940f0b92"}, + {file = "mypy-1.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:091f53ff88cb093dcc33c29eee522c087a438df65eb92acd371161c1f4380ff0"}, + {file = "mypy-1.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb7ff4007865833c470a601498ba30462b7374342580e2346bf7884557e40531"}, + {file = "mypy-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49499cf1e464f533fc45be54d20a6351a312f96ae7892d8e9f1708140e27ce41"}, + {file = "mypy-1.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4c192445899c69f07874dabda7e931b0cc811ea055bf82c1ababf358b9b2a72c"}, + {file = "mypy-1.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:3df87094028e52766b0a59a3e46481bb98b27986ed6ded6a6cc35ecc75bb9182"}, + {file = "mypy-1.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c8835a07b8442da900db47ccfda76c92c69c3a575872a5b764332c4bacb5a0a"}, + {file = "mypy-1.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:24f3de8b9e7021cd794ad9dfbf2e9fe3f069ff5e28cb57af6f873ffec1cb0425"}, + {file = "mypy-1.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:856bad61ebc7d21dbc019b719e98303dc6256cec6dcc9ebb0b214b81d6901bd8"}, + {file = "mypy-1.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:89513ddfda06b5c8ebd64f026d20a61ef264e89125dc82633f3c34eeb50e7d60"}, + {file = "mypy-1.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:9f8464ed410ada641c29f5de3e6716cbdd4f460b31cf755b2af52f2d5ea79ead"}, + {file = "mypy-1.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:971104bcb180e4fed0d7bd85504c9036346ab44b7416c75dd93b5c8c6bb7e28f"}, + {file = "mypy-1.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab98b8f6fdf669711f3abe83a745f67f50e3cbaea3998b90e8608d2b459fd566"}, + {file = "mypy-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a69db3018b87b3e6e9dd28970f983ea6c933800c9edf8c503c3135b3274d5ad"}, + {file = "mypy-1.6.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:dccd850a2e3863891871c9e16c54c742dba5470f5120ffed8152956e9e0a5e13"}, + {file = "mypy-1.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:f8598307150b5722854f035d2e70a1ad9cc3c72d392c34fffd8c66d888c90f17"}, + {file = "mypy-1.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fea451a3125bf0bfe716e5d7ad4b92033c471e4b5b3e154c67525539d14dc15a"}, + {file = "mypy-1.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e28d7b221898c401494f3b77db3bac78a03ad0a0fff29a950317d87885c655d2"}, + {file = "mypy-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4b7a99275a61aa22256bab5839c35fe8a6887781862471df82afb4b445daae6"}, + {file = "mypy-1.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7469545380dddce5719e3656b80bdfbb217cfe8dbb1438532d6abc754b828fed"}, + {file = "mypy-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:7807a2a61e636af9ca247ba8494031fb060a0a744b9fee7de3a54bed8a753323"}, + {file = "mypy-1.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d2dad072e01764823d4b2f06bc7365bb1d4b6c2f38c4d42fade3c8d45b0b4b67"}, + {file = "mypy-1.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b19006055dde8a5425baa5f3b57a19fa79df621606540493e5e893500148c72f"}, + {file = "mypy-1.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31eba8a7a71f0071f55227a8057468b8d2eb5bf578c8502c7f01abaec8141b2f"}, + {file = "mypy-1.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e0db37ac4ebb2fee7702767dfc1b773c7365731c22787cb99f507285014fcaf"}, + {file = "mypy-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:c69051274762cccd13498b568ed2430f8d22baa4b179911ad0c1577d336ed849"}, + {file = "mypy-1.6.0-py3-none-any.whl", hash = "sha256:9e1589ca150a51d9d00bb839bfeca2f7a04f32cd62fad87a847bc0818e15d7dc"}, + {file = "mypy-1.6.0.tar.gz", hash = "sha256:4f3d27537abde1be6d5f2c96c29a454da333a2a271ae7d5bc7110e6d4b7beb3f"}, ] [package.dependencies] @@ -519,13 +519,13 @@ setuptools = "*" [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -541,13 +541,13 @@ files = [ [[package]] name = "platformdirs" -version = "3.10.0" +version = "3.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, - {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, ] [package.extras] @@ -946,6 +946,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -953,8 +954,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -971,6 +979,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -978,6 +987,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1210,13 +1220,13 @@ files = [ [[package]] name = "types-tzlocal" -version = "5.0.1.1" +version = "5.1.0.0" description = "Typing stubs for tzlocal" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "types-tzlocal-5.0.1.1.tar.gz", hash = "sha256:d0a9fb6f846c0416f18a040f2d15714d3d52a6165c541e9f4eef7356b53af011"}, - {file = "types_tzlocal-5.0.1.1-py3-none-any.whl", hash = "sha256:55a66e06e0332d6e04886cf578953dcc465b3054e3f0a337da3bdcbaa59122a4"}, + {file = "types-tzlocal-5.1.0.0.tar.gz", hash = "sha256:44c5f3497283070b899152f307385b5cf7f7d9b91b4b6aef0dc1406d897808bd"}, + {file = "types_tzlocal-5.1.0.0-py3-none-any.whl", hash = "sha256:6aaf72ebf3d1c5a6711b7b47c434a5f4697efc6aac79d4a99e1fc24cc2b07863"}, ] [package.dependencies] @@ -1246,13 +1256,13 @@ files = [ [[package]] name = "tzlocal" -version = "5.0.1" +version = "5.1" description = "tzinfo object for the local timezone" optional = false python-versions = ">=3.7" files = [ - {file = "tzlocal-5.0.1-py3-none-any.whl", hash = "sha256:f3596e180296aaf2dbd97d124fe76ae3a0e3d32b258447de7b939b3fd4be992f"}, - {file = "tzlocal-5.0.1.tar.gz", hash = "sha256:46eb99ad4bdb71f3f72b7d24f4267753e240944ecfc16f25d2719ba89827a803"}, + {file = "tzlocal-5.1-py3-none-any.whl", hash = "sha256:2938498395d5f6a898ab8009555cb37a4d360913ad375d4747ef16826b03ef23"}, + {file = "tzlocal-5.1.tar.gz", hash = "sha256:a5ccb2365b295ed964e0a98ad076fe10c495591e75505d34f154d60a7f1ed722"}, ] [package.dependencies] @@ -1390,4 +1400,4 @@ zmq = ["pyzmq"] [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "3ec5450a6245f5beceb41aeeb47ba722ab06e15e17fbd0432ad71cee6ee52bac" +content-hash = "f3bf5bd3992cd049af66ebdd02e5ee448ac7008d1e2daeea46823bf86001f1c8" diff --git a/pyproject.toml b/pyproject.toml index a2ed4f2..9031103 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ freezegun = "^1.2.2" pytest-mock = "^3.11.1" tzlocal = "^5.0.1" types-tzlocal = "^5.0.1.1" +types-pytz = "^2023.3.1.1" [tool.poetry.extras] zmq = ["pyzmq"] diff --git a/taskiq/abc/broker.py b/taskiq/abc/broker.py index b935e38..234c2ca 100644 --- a/taskiq/abc/broker.py +++ b/taskiq/abc/broker.py @@ -25,12 +25,14 @@ from typing_extensions import ParamSpec, Self, TypeAlias from taskiq.abc.middleware import TaskiqMiddleware +from taskiq.abc.serializer import TaskiqSerializer from taskiq.acks import AckableMessage from taskiq.decor import AsyncTaskiqDecoratedTask from taskiq.events import TaskiqEvents -from taskiq.formatters.json_formatter import JSONFormatter +from taskiq.formatters.proxy_formatter import ProxyFormatter from taskiq.message import BrokerMessage from taskiq.result_backends.dummy import DummyResultBackend +from taskiq.serializers.json_serializer import JSONSerializer from taskiq.state import TaskiqState from taskiq.utils import maybe_awaitable, remove_suffix from taskiq.warnings import TaskiqDeprecationWarning @@ -97,7 +99,8 @@ def __init__( self.middlewares: "List[TaskiqMiddleware]" = [] self.result_backend = result_backend self.decorator_class = AsyncTaskiqDecoratedTask - self.formatter: "TaskiqFormatter" = JSONFormatter() + self.serializer: TaskiqSerializer = JSONSerializer() + self.formatter: "TaskiqFormatter" = ProxyFormatter(self) self.id_generator = task_id_generator self.local_task_registry: Dict[str, AsyncTaskiqDecoratedTask[Any, Any]] = {} # Every event has a list of handlers. @@ -479,6 +482,19 @@ def with_event_handlers( self.event_handlers[event].extend(handlers) return self + def with_serializer( + self, + serializer: TaskiqSerializer, + ) -> "Self": # pragma: no cover + """ + Set a new serializer and return an updated broker. + + :param serializer: new serializer. + :return: self + """ + self.serializer = serializer + return self + def _register_task( self, task_name: str, diff --git a/taskiq/abc/serializer.py b/taskiq/abc/serializer.py new file mode 100644 index 0000000..4f53ccf --- /dev/null +++ b/taskiq/abc/serializer.py @@ -0,0 +1,24 @@ +from abc import ABC, abstractmethod +from typing import Any + + +class TaskiqSerializer(ABC): + """Custom serializer for brokers.""" + + @abstractmethod + def dumpb(self, value: Any) -> bytes: + """ + Dump value to bytes for sending through the wire. + + :param value: value to encode. + :return: encoded value. + """ + + @abstractmethod + def loadb(self, value: bytes) -> Any: + """ + Parse byte-encoded value received from the wire. + + :param message: value to parse. + :return: decoded value. + """ diff --git a/taskiq/compat.py b/taskiq/compat.py index 2939c5f..48312fd 100644 --- a/taskiq/compat.py +++ b/taskiq/compat.py @@ -16,6 +16,15 @@ def parse_obj_as(annot: T, obj: Any) -> T: return pydantic.TypeAdapter(annot).validate_python(obj) + def model_validate( + model_class: Type[Model], + message: Dict[str, Any], + ) -> Model: + return model_class.model_validate(message) + + def model_dump(instance: Model) -> Dict[str, Any]: + return instance.model_dump() + def model_validate_json( model_class: Type[Model], message: Union[str, bytes, bytearray], @@ -37,6 +46,15 @@ def model_copy( else: parse_obj_as = pydantic.parse_obj_as # type: ignore + def model_validate( + model_class: Type[Model], + message: Dict[str, Any], + ) -> Model: + return model_class.parse_obj(message) + + def model_dump(instance: Model) -> Dict[str, Any]: + return instance.dict() + def model_validate_json( model_class: Type[Model], message: Union[str, bytes, bytearray], diff --git a/taskiq/formatters/json_formatter.py b/taskiq/formatters/json_formatter.py index fa7baf0..3d49e28 100644 --- a/taskiq/formatters/json_formatter.py +++ b/taskiq/formatters/json_formatter.py @@ -4,7 +4,7 @@ class JSONFormatter(TaskiqFormatter): - """Default taskiq formatter.""" + """JSON taskiq formatter.""" def dumps(self, message: TaskiqMessage) -> BrokerMessage: """ diff --git a/taskiq/formatters/proxy_formatter.py b/taskiq/formatters/proxy_formatter.py new file mode 100644 index 0000000..a220871 --- /dev/null +++ b/taskiq/formatters/proxy_formatter.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +from taskiq.abc.formatter import TaskiqFormatter +from taskiq.compat import model_dump, model_validate +from taskiq.message import BrokerMessage, TaskiqMessage + +if TYPE_CHECKING: + from taskiq.abc.broker import AsyncBroker + + +class ProxyFormatter(TaskiqFormatter): + """Default taskiq formatter.""" + + def __init__(self, broker: "AsyncBroker") -> None: + self.broker = broker + + def dumps(self, message: TaskiqMessage) -> BrokerMessage: + """ + Dumps taskiq message to some broker message format. + + :param message: message to send. + :return: Dumped message. + """ + return BrokerMessage( + task_id=message.task_id, + task_name=message.task_name, + message=self.broker.serializer.dumpb(model_dump(message)), + labels=message.labels, + ) + + def loads(self, message: bytes) -> TaskiqMessage: + """ + Loads json from message. + + :param message: broker's message. + :return: parsed taskiq message. + """ + return model_validate(TaskiqMessage, self.broker.serializer.loadb(message)) diff --git a/taskiq/message.py b/taskiq/message.py index 6cc0468..129073f 100644 --- a/taskiq/message.py +++ b/taskiq/message.py @@ -14,7 +14,7 @@ class TaskiqMessage(BaseModel): task_id: str task_name: str - labels: Dict[str, str] + labels: Dict[str, Any] args: List[Any] kwargs: Dict[str, Any] @@ -25,4 +25,4 @@ class BrokerMessage(BaseModel): task_id: str task_name: str message: bytes - labels: Dict[str, str] + labels: Dict[str, Any] diff --git a/taskiq/serializers/__init__.py b/taskiq/serializers/__init__.py new file mode 100644 index 0000000..2f24f33 --- /dev/null +++ b/taskiq/serializers/__init__.py @@ -0,0 +1 @@ +"""Taskiq serializers.""" diff --git a/taskiq/serializers/json_serializer.py b/taskiq/serializers/json_serializer.py new file mode 100644 index 0000000..e7d8d38 --- /dev/null +++ b/taskiq/serializers/json_serializer.py @@ -0,0 +1,26 @@ +from json import dumps, loads +from typing import Any + +from taskiq.abc.serializer import TaskiqSerializer + + +class JSONSerializer(TaskiqSerializer): + """Default taskiq serizalizer.""" + + def dumpb(self, value: Any) -> bytes: + """ + Dumps taskiq message to some broker message format. + + :param message: message to send. + :return: Dumped message. + """ + return dumps(value).encode() + + def loadb(self, value: bytes) -> Any: + """ + Parse byte-encoded value received from the wire. + + :param message: value to parse. + :return: decoded value. + """ + return loads(value.decode()) diff --git a/tests/formatters/test_json_formatter.py b/tests/formatters/test_json_formatter.py new file mode 100644 index 0000000..360b112 --- /dev/null +++ b/tests/formatters/test_json_formatter.py @@ -0,0 +1,45 @@ +import pytest + +from taskiq.formatters.json_formatter import JSONFormatter +from taskiq.message import BrokerMessage, TaskiqMessage + + +@pytest.mark.anyio +async def test_json_dumps() -> None: + fmt = JSONFormatter() + msg = TaskiqMessage( + task_id="task-id", + task_name="task.name", + labels={"label1": 1, "label2": "text"}, + args=[1, "a"], + kwargs={"p1": "v1"}, + ) + expected = BrokerMessage( + task_id="task-id", + task_name="task.name", + message=( + b'{"task_id":"task-id","task_name":"task.name",' + b'"labels":{"label1":1,"label2":"text"},' + b'"args":[1,"a"],"kwargs":{"p1":"v1"}}' + ), + labels={"label1": 1, "label2": "text"}, + ) + assert fmt.dumps(msg) == expected + + +@pytest.mark.anyio +async def test_json_loads() -> None: + fmt = JSONFormatter() + msg = ( + b'{"task_id":"task-id","task_name":"task.name",' + b'"labels":{"label1":1,"label2":"text"},' + b'"args":[1,"a"],"kwargs":{"p1":"v1"}}' + ) + expected = TaskiqMessage( + task_id="task-id", + task_name="task.name", + labels={"label1": 1, "label2": "text"}, + args=[1, "a"], + kwargs={"p1": "v1"}, + ) + assert fmt.loads(msg) == expected diff --git a/tests/formatters/test_proxy_formatter.py b/tests/formatters/test_proxy_formatter.py new file mode 100644 index 0000000..50d179a --- /dev/null +++ b/tests/formatters/test_proxy_formatter.py @@ -0,0 +1,47 @@ +import pytest + +from taskiq.brokers.inmemory_broker import InMemoryBroker +from taskiq.message import BrokerMessage, TaskiqMessage + + +@pytest.mark.anyio +async def test_proxy_dumps() -> None: + # uses json serializer by default + broker = InMemoryBroker() + msg = TaskiqMessage( + task_id="task-id", + task_name="task.name", + labels={"label1": 1, "label2": "text"}, + args=[1, "a"], + kwargs={"p1": "v1"}, + ) + expected = BrokerMessage( + task_id="task-id", + task_name="task.name", + message=( + b'{"task_id": "task-id", "task_name": "task.name", ' + b'"labels": {"label1": 1, "label2": "text"}, ' + b'"args": [1, "a"], "kwargs": {"p1": "v1"}}' + ), + labels={"label1": 1, "label2": "text"}, + ) + assert broker.formatter.dumps(msg) == expected + + +@pytest.mark.anyio +async def test_proxy_loads() -> None: + # uses json serializer by default + broker = InMemoryBroker() + msg = ( + b'{"task_id":"task-id","task_name":"task.name",' + b'"labels":{"label1":1,"label2":"text"},' + b'"args":[1,"a"],"kwargs":{"p1":"v1"}}' + ) + expected = TaskiqMessage( + task_id="task-id", + task_name="task.name", + labels={"label1": 1, "label2": "text"}, + args=[1, "a"], + kwargs={"p1": "v1"}, + ) + assert broker.formatter.loads(msg) == expected diff --git a/tests/serializers/test_json_serializer.py b/tests/serializers/test_json_serializer.py new file mode 100644 index 0000000..7685185 --- /dev/null +++ b/tests/serializers/test_json_serializer.py @@ -0,0 +1,23 @@ +import pytest + +from taskiq.serializers.json_serializer import JSONSerializer + + +@pytest.mark.anyio +async def test_json_dumpb() -> None: + serizalizer = JSONSerializer() + assert serizalizer.dumpb(None) == b"null" # noqa: PLR2004 + assert serizalizer.dumpb(1) == b"1" # noqa: PLR2004 + assert serizalizer.dumpb("a") == b'"a"' # noqa: PLR2004 + assert serizalizer.dumpb(["a"]) == b'["a"]' # noqa: PLR2004 + assert serizalizer.dumpb({"a": "b"}) == b'{"a": "b"}' # noqa: PLR2004 + + +@pytest.mark.anyio +async def test_json_loadb() -> None: + serizalizer = JSONSerializer() + assert serizalizer.loadb(b"null") is None + assert serizalizer.loadb(b"1") == 1 + assert serizalizer.loadb(b'"a"') == "a" + assert serizalizer.loadb(b'["a"]') == ["a"] + assert serizalizer.loadb(b'{"a": "b"}') == {"a": "b"}