From 27bb2c08ee69ccfbe8a0901c812364e92dae68af Mon Sep 17 00:00:00 2001 From: Matt Bullock Date: Mon, 31 Aug 2020 09:47:05 -0700 Subject: [PATCH] chore: fix all CI and start migration to GitHub Actions (#139) * chore: force tox to update pip * chore: update isort configuration to 5.0.0 * chore: autoformat * chore: hypothesis.HealthCheck.hung_test is deprecated * chore: flake8 linting * chore: update pylint and flake8 configs * chore: linting fixes * chore: update default Python envlist - remove 3.4 - add 3.8 * chore: Python 2 lists do not have copy() * chore: address all pylint issues aside from TODO references * chore: unlock mypy version * chore: fix type annotation syntax errors * chore: move TODOs into GitHub issues * chore: move test TODOs to GitHub issues * chore: autoformat * chore: rework moto use - fixes issues with multiple service mocks in Python 2 - module scope avoids resetting the mocked service for tests that use multiple mocked tables * chore: force nocmk environment to black all environment variables * chore: add GitHub Actions workflows * chore: move sourcebuildcheck and nocmk into upstream-py3 job * chore: add ci-requirements.txt * chore: work around bug in Python 2 Hypothesis behavior by only running fast tests for Python 2 * chore: fix sourcebuildcheck script - The ls command was getting a relative path when the script ran but a bare filename in manual checks. * chore: pruning known runs from Travis that fail due to known infrastructure issues --- .github/workflows/ci_static-analysis.yaml | 44 ++++++ .github/workflows/ci_tests.yaml | 131 ++++++++++++++++++ .travis.yml | 17 +-- ci-requirements.txt | 1 + setup.cfg | 3 +- setup.py | 2 +- .../encrypted/client.py | 3 +- src/dynamodb_encryption_sdk/encrypted/item.py | 2 +- .../encrypted/resource.py | 4 +- .../encrypted/table.py | 2 +- src/dynamodb_encryption_sdk/identifiers.py | 4 +- .../internal/crypto/authentication.py | 12 +- .../internal/crypto/encryption.py | 7 +- .../crypto/jce_bridge/authentication.py | 29 ++-- .../internal/crypto/jce_bridge/encryption.py | 6 +- .../internal/crypto/jce_bridge/primitives.py | 18 +-- .../internal/dynamodb_types.py | 10 +- .../formatting/deserialize/attribute.py | 47 +++++-- .../formatting/material_description.py | 10 +- .../formatting/serialize/attribute.py | 37 +++-- .../internal/identifiers.py | 2 +- src/dynamodb_encryption_sdk/internal/utils.py | 18 ++- .../material_providers/aws_kms.py | 4 +- .../material_providers/most_recent.py | 11 +- .../material_providers/store/__init__.py | 2 +- .../material_providers/store/meta.py | 11 +- .../materials/__init__.py | 1 + .../materials/wrapped.py | 7 +- src/dynamodb_encryption_sdk/structures.py | 2 +- src/dynamodb_encryption_sdk/transform.py | 6 +- src/pylintrc | 5 +- test/acceptance/acceptance_test_utils.py | 4 +- test/functional/delegated_keys/test_jce.py | 2 +- test/functional/encrypted/test_client.py | 12 +- test/functional/encrypted/test_resource.py | 10 +- test/functional/encrypted/test_table.py | 16 ++- test/functional/functional_test_utils.py | 54 ++++---- test/functional/hypothesis_strategies.py | 8 +- .../material_providers/store/test_meta.py | 7 +- .../material_providers/test_most_recent.py | 10 +- test/functional/materials/test_raw.py | 4 +- test/functional/test_structures.py | 19 ++- test/integration/integration_test_utils.py | 7 +- .../material_providers/store/test_meta.py | 2 +- .../material_providers/test_most_recent.py | 8 +- test/pylintrc | 5 +- test/requirements.txt | 3 +- test/source-build-check.sh | 26 ++-- .../internal/formatting/test_attribute.py | 7 +- test/unit/material_providers/test_aws_kms.py | 3 - tox.ini | 32 +++-- 51 files changed, 473 insertions(+), 224 deletions(-) create mode 100644 .github/workflows/ci_static-analysis.yaml create mode 100644 .github/workflows/ci_tests.yaml create mode 100644 ci-requirements.txt diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml new file mode 100644 index 00000000..b5d52954 --- /dev/null +++ b/.github/workflows/ci_static-analysis.yaml @@ -0,0 +1,44 @@ +# This workflow runs static analysis checks on pull requests. +name: static analysis + +on: + pull_request: + push: + # Run once a day + schedule: + - cron: '0 0 * * *' + +jobs: + analysis: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + category: +# Disabled pending completion of integration +# https://github.com/aws/aws-dynamodb-encryption-python/issues/66 +# - mypy-py2 +# - mypy-py3 + - bandit + - doc8 + - readme + - docs + - flake8 + - pylint + - flake8-tests + - flake8-examples + - pylint-tests + - pylint-examples + - black-check + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.x + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: check + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml new file mode 100644 index 00000000..b94c4f9e --- /dev/null +++ b/.github/workflows/ci_tests.yaml @@ -0,0 +1,131 @@ +# This workflow runs tests on pull requests. +name: tests + +on: + pull_request: + push: + # Run once a day + schedule: + - cron: '0 0 * * *' + +jobs: + # Hypothesis no longer supports Python 2 and + # there is a bug that appears with our slow tests + # only on Python 2. + # Until we also drop Python 2 support, + # the workaround is just that we don't run the slow tests + # on Python 2. + py2-tests: + runs-on: ${{ matrix.platform.os }} + strategy: + fail-fast: true + matrix: + platform: + - os: ubuntu-latest + architecture: x64 + - os: windows-latest + architecture: x64 + # x86 builds are only meaningful for Windows + - os: windows-latest + architecture: x86 + - os: macos-latest + architecture: x64 + category: + - local-fast + # These require credentials. + # Enable them once we sort how to provide them. + # - integ-fast + # - examples + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 2.7 + architecture: ${{ matrix.platform.architecture }} + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv + tests: + runs-on: ${{ matrix.platform.os }} + strategy: + fail-fast: true + matrix: + platform: + - os: ubuntu-latest + architecture: x64 + - os: windows-latest + architecture: x64 + # x86 builds are only meaningful for Windows + - os: windows-latest + architecture: x86 + - os: macos-latest + architecture: x64 + python: + - 3.5 + - 3.6 + - 3.7 + - 3.8 + - 3.x + category: + - local-slow +# These require credentials. +# Enable them once we sort how to provide them. +# - integ-slow +# - examples + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python }} + architecture: ${{ matrix.platform.architecture }} + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv + upstream-py3: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + category: + - nocmk + - sourcebuildcheck + - test-upstream-requirements-py37 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv + upstream-py2: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + category: + - test-upstream-requirements-py27 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 2.7 + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: ${{ matrix.category }} + run: tox -- -vv diff --git a/.travis.yml b/.travis.yml index 070d4a39..df8e4800 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,20 +2,17 @@ sudo: false language: python matrix: include: + # Hypothesis no longer supports Python 2 and + # there is a bug that appears with our slow tests + # only on Python 2. + # Until we also drop Python 2 support, + # the workaround is just that we don't run the slow tests + # on Python 2. # CPython 2.7 - python: 2.7 - env: TOXENV=py27-travis-local-slow + env: TOXENV=py27-travis-local-fast - python: 2.7 env: TOXENV=py27-travis-integ-slow - - python: 2.7 - env: TOXENV=py27-travis-isolation - # CPython 3.4 - - python: 3.4 - env: TOXENV=py34-travis-local-slow - - python: 3.4 - env: TOXENV=py34-travis-integ-slow - - python: 3.4 - env: TOXENV=py34-travis-isolation # CPython 3.5 - python: 3.5 env: TOXENV=py35-travis-local-slow diff --git a/ci-requirements.txt b/ci-requirements.txt new file mode 100644 index 00000000..053148f8 --- /dev/null +++ b/ci-requirements.txt @@ -0,0 +1 @@ +tox diff --git a/setup.cfg b/setup.cfg index 5949c4dd..b693d2e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ log_level=DEBUG # Flake8 Configuration [flake8] -max_complexity = 10 +max_complexity = 11 max_line_length = 120 import_order_style = google application_import_names = dynamodb_encryption_sdk @@ -61,6 +61,5 @@ multi_line_output = 3 include_trailing_comma = True force_grid_wrap = 0 combine_as_imports = True -not_skip = __init__.py known_first_party = dynamodb_encryption_sdk known_third_party =attr,aws_kms_encrypted_client,aws_kms_encrypted_item,aws_kms_encrypted_resource,aws_kms_encrypted_table,boto3,botocore,cryptography,dynamodb_encryption_sdk,functional_test_utils,functional_test_vector_generators,hypothesis,hypothesis_strategies,integration_test_utils,mock,most_recent_provider_encrypted_table,moto,mypy_extensions,pytest,pytest_mock,setuptools,six,wrapped_rsa_encrypted_table,wrapped_symmetric_encrypted_table diff --git a/setup.py b/setup.py index a1c79400..2dc604cd 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def get_version(): def get_requirements(): """Reads the requirements file.""" requirements = read("requirements.txt") - return [r for r in requirements.strip().splitlines()] + return requirements.strip().splitlines() setup( diff --git a/src/dynamodb_encryption_sdk/encrypted/client.py b/src/dynamodb_encryption_sdk/encrypted/client.py index ebee5d90..bd8f2c58 100644 --- a/src/dynamodb_encryption_sdk/encrypted/client.py +++ b/src/dynamodb_encryption_sdk/encrypted/client.py @@ -96,7 +96,8 @@ def __getattr__(self, name): def paginate(self, **kwargs): # type: (**Any) -> Iterator[Dict] - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Create an iterator that will paginate through responses from the underlying paginator, transparently decrypting any returned items. """ diff --git a/src/dynamodb_encryption_sdk/encrypted/item.py b/src/dynamodb_encryption_sdk/encrypted/item.py index 25f1ef50..b491a34a 100644 --- a/src/dynamodb_encryption_sdk/encrypted/item.py +++ b/src/dynamodb_encryption_sdk/encrypted/item.py @@ -171,7 +171,7 @@ def decrypt_dynamodb_item(item, crypto_config): :rtype: dict """ unique_actions = set([crypto_config.attribute_actions.default_action.name]) - unique_actions.update(set([action.name for action in crypto_config.attribute_actions.attribute_actions.values()])) + unique_actions.update({action.name for action in crypto_config.attribute_actions.attribute_actions.values()}) if crypto_config.attribute_actions.take_no_actions: # If we explicitly have been told not to do anything to this item, just copy it. diff --git a/src/dynamodb_encryption_sdk/encrypted/resource.py b/src/dynamodb_encryption_sdk/encrypted/resource.py index 58b5c776..b5b71f8b 100644 --- a/src/dynamodb_encryption_sdk/encrypted/resource.py +++ b/src/dynamodb_encryption_sdk/encrypted/resource.py @@ -41,7 +41,7 @@ @attr.s(init=False) class EncryptedTablesCollectionManager(object): - # pylint: disable=too-few-public-methods + # pylint: disable=too-few-public-methods,too-many-instance-attributes """Tables collection manager that provides :class:`EncryptedTable` objects. https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.tables @@ -119,7 +119,7 @@ def _transform_table(self, method, **kwargs): @attr.s(init=False) class EncryptedResource(object): - # pylint: disable=too-few-public-methods + # pylint: disable=too-few-public-methods,too-many-instance-attributes """High-level helper class to provide a familiar interface to encrypted tables. >>> import boto3 diff --git a/src/dynamodb_encryption_sdk/encrypted/table.py b/src/dynamodb_encryption_sdk/encrypted/table.py index 51c69b74..128cb896 100644 --- a/src/dynamodb_encryption_sdk/encrypted/table.py +++ b/src/dynamodb_encryption_sdk/encrypted/table.py @@ -42,7 +42,7 @@ @attr.s(init=False) class EncryptedTable(object): - # pylint: disable=too-few-public-methods + # pylint: disable=too-few-public-methods,too-many-instance-attributes """High-level helper class to provide a familiar interface to encrypted tables. >>> import boto3 diff --git a/src/dynamodb_encryption_sdk/identifiers.py b/src/dynamodb_encryption_sdk/identifiers.py index 9187ecb9..e0156b06 100644 --- a/src/dynamodb_encryption_sdk/identifiers.py +++ b/src/dynamodb_encryption_sdk/identifiers.py @@ -35,12 +35,12 @@ def __gt__(self, other): def __lt__(self, other): # type: (CryptoAction) -> bool """Define CryptoAction equality.""" - return self.value < other.value + return self.value < other.value # pylint: disable=comparison-with-callable def __eq__(self, other): # type: (CryptoAction) -> bool """Define CryptoAction equality.""" - return self.value == other.value + return self.value == other.value # pylint: disable=comparison-with-callable class EncryptionKeyType(Enum): diff --git a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py index c45ead11..d5247688 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py @@ -28,6 +28,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Text # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks @@ -55,7 +56,8 @@ def sign_item(encrypted_item, signing_key, crypto_config): attribute_actions=crypto_config.attribute_actions, ), ) - return {Tag.BINARY.dynamodb_tag: signature} + # for some reason pylint can't follow the Enum member attributes + return {Tag.BINARY.dynamodb_tag: signature} # pylint: disable=no-member def verify_item_signature(signature_attribute, encrypted_item, verification_key, crypto_config): @@ -67,7 +69,8 @@ def verify_item_signature(signature_attribute, encrypted_item, verification_key, :param DelegatedKey verification_key: DelegatedKey to use to calculate the signature :param CryptoConfig crypto_config: Cryptographic configuration """ - signature = signature_attribute[Tag.BINARY.dynamodb_tag] + # for some reason pylint can't follow the Enum member attributes + signature = signature_attribute[Tag.BINARY.dynamodb_tag] # pylint: disable=no-member verification_key.verify( algorithm=verification_key.algorithm, signature=signature, @@ -97,10 +100,11 @@ def _string_to_sign(item, table_name, attribute_actions): data_to_sign.extend(_hash_data(hasher=hasher, data=key.encode(TEXT_ENCODING))) + # for some reason pylint can't follow the Enum member attributes if action is CryptoAction.SIGN_ONLY: - data_to_sign.extend(SignatureValues.PLAINTEXT.sha256) + data_to_sign.extend(SignatureValues.PLAINTEXT.sha256) # pylint: disable=no-member else: - data_to_sign.extend(SignatureValues.ENCRYPTED.sha256) + data_to_sign.extend(SignatureValues.ENCRYPTED.sha256) # pylint: disable=no-member data_to_sign.extend(_hash_data(hasher=hasher, data=serialize_attribute(item[key]))) return bytes(data_to_sign) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py index 8283c82a..3737d520 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py @@ -18,6 +18,7 @@ """ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Text # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks @@ -46,7 +47,8 @@ def encrypt_attribute(attribute_name, attribute, encryption_key, algorithm): encrypted_attribute = encryption_key.encrypt( algorithm=algorithm, name=attribute_name, plaintext=serialized_attribute ) - return {Tag.BINARY.dynamodb_tag: encrypted_attribute} + # for some reason pylint can't follow the Enum member attributes + return {Tag.BINARY.dynamodb_tag: encrypted_attribute} # pylint: disable=no-member def decrypt_attribute(attribute_name, attribute, decryption_key, algorithm): @@ -60,7 +62,8 @@ def decrypt_attribute(attribute_name, attribute, decryption_key, algorithm): :returns: Plaintext DynamoDB attribute :rtype: dict """ - encrypted_attribute = attribute[Tag.BINARY.dynamodb_tag] + # for some reason pylint can't follow the Enum member attributes + encrypted_attribute = attribute[Tag.BINARY.dynamodb_tag] # pylint: disable=no-member decrypted_attribute = decryption_key.decrypt( algorithm=algorithm, name=attribute_name, ciphertext=encrypted_attribute ) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index a40c4a62..b2244ac7 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -49,7 +49,8 @@ class JavaAuthenticator(object): @abc.abstractmethod def load_key(self, key, key_type, key_encoding): # (bytes, EncryptionKeyType, KeyEncodingType) -> Any - # TODO: narrow down the output type + # narrow down the output type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Load a key from bytes. :param bytes key: Raw key bytes to load @@ -104,10 +105,8 @@ class JavaMac(JavaAuthenticator): algorithm_type = attr.ib(validator=callable_validator) hash_type = attr.ib(validator=callable_validator) - def __init__( - self, java_name, algorithm_type, hash_type # type: Text # type: Callable # type: Callable - ): # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, algorithm_type, hash_type): # noqa=D107 + # type: (Text, Callable, Callable) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -139,7 +138,7 @@ def load_key(self, key, key_type, key_encoding): raise ValueError("Key type must be symmetric and encoding must be raw.") if len(key) * 8 < MinimumKeySizes.HMAC.value: - _LOGGER.warning("HMAC keys smaller than %d bits are unsafe" % MinimumKeySizes.HMAC.value) + _LOGGER.warning("HMAC keys smaller than %d bits are unsafe", MinimumKeySizes.HMAC.value) return key @@ -208,10 +207,8 @@ class JavaSignature(JavaAuthenticator): hash_type = attr.ib(validator=callable_validator) padding_type = attr.ib(validator=callable_validator) - def __init__( - self, java_name, algorithm_type, hash_type, padding_type # type: Text # type: Callable # type: Callable - ): # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, algorithm_type, hash_type, padding_type): # noqa=D107 + # type: (Text, Any, Callable, Callable) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -237,25 +234,25 @@ def validate_algorithm(self, algorithm): def load_key(self, key, key_type, key_encoding): # (bytes, EncryptionKeyType, KeyEncodingType) -> Any - # TODO: narrow down the output type + # narrow down the output type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Load a key object from the provided raw key bytes. :param bytes key: Raw key bytes to load :param EncryptionKeyType key_type: Type of key to load :param KeyEncodingType key_encoding: Encoding used to serialize ``key`` :returns: Loaded key - :rtype: TODO: :raises ValueError: if ``key_type`` and ``key_encoding`` are not a valid pairing """ return load_rsa_key(key, key_type, key_encoding) def sign(self, key, data): # type: (Any, bytes) -> bytes - # TODO: narrow down the key type + # narrow down the key type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Sign ``data`` using loaded ``key``. :param key: Loaded key - :type key: TODO: :param bytes data: Data to sign :returns: Calculated signature :rtype: bytes @@ -272,11 +269,11 @@ def sign(self, key, data): def verify(self, key, signature, data): # type: (Any, bytes, bytes) -> None - # TODO: narrow down the key type + # narrow down the key type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Verify ``signature`` over ``data`` using ``key``. :param key: Loaded key - :type key: TODO: :param bytes signature: Signature to verify :param bytes data: Data over which to verify signature :raises SignatureVerificationError: if unable to verify ``signature`` diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py index b09f60a1..6a123597 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py @@ -47,10 +47,8 @@ class JavaCipher(object): mode = attr.ib(validator=attr.validators.instance_of(JavaMode)) padding = attr.ib(validator=attr.validators.instance_of(JavaPadding)) - def __init__( - self, cipher, mode, padding # type: JavaEncryptionAlgorithm # type: JavaMode # type: JavaPadding - ): # noqa=D107 - # type: (...) -> None + def __init__(self, cipher, mode, padding): # noqa=D107 + # type: (JavaEncryptionAlgorithm, JavaMode, JavaPadding) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py index 29497fc9..564bed80 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py @@ -121,8 +121,7 @@ class SimplePadding(JavaPadding): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) padding = attr.ib(validator=callable_validator) - def __init__(self, java_name, padding): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, padding): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -148,8 +147,7 @@ class BlockSizePadding(JavaPadding): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) padding = attr.ib(validator=callable_validator) - def __init__(self, java_name, padding): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, padding): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -227,8 +225,7 @@ class JavaMode(object): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) mode = attr.ib(validator=callable_validator) - def __init__(self, java_name, mode): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, mode): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -256,8 +253,7 @@ class JavaEncryptionAlgorithm(object): java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) cipher = attr.ib() - def __init__(self, java_name, cipher): # type: Text # type: Callable # noqa=D107 - # type: (...) -> None + def __init__(self, java_name, cipher): # type: (Text, Callable) -> None # noqa=D107 # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 @@ -433,14 +429,14 @@ def decrypt(self, key, data, mode, padding): def load_rsa_key(key, key_type, key_encoding): # (bytes, EncryptionKeyType, KeyEncodingType) -> Any - # TODO: narrow down the output type + # narrow down the output type + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Load an RSA key object from the provided raw key bytes. :param bytes key: Raw key bytes to load :param EncryptionKeyType key_type: Type of key to load :param KeyEncodingType key_encoding: Encoding used to serialize ``key`` :returns: Loaded key - :rtype: TODO: :raises ValueError: if ``key_type`` and ``key_encoding`` are not a valid pairing """ try: @@ -455,7 +451,7 @@ def load_rsa_key(key, key_type, key_encoding): loaded_key = loader(**kwargs) if loaded_key.key_size < MinimumKeySizes.RSA.value: - _LOGGER.warning("RSA keys smaller than %d bits are unsafe" % MinimumKeySizes.RSA.value) + _LOGGER.warning("RSA keys smaller than %d bits are unsafe", MinimumKeySizes.RSA.value) return loaded_key diff --git a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py index 1096a645..0509a59b 100644 --- a/src/dynamodb_encryption_sdk/internal/dynamodb_types.py +++ b/src/dynamodb_encryption_sdk/internal/dynamodb_types.py @@ -4,16 +4,20 @@ No guarantee is provided on the modules and APIs within this namespace staying consistent. Directly reference at your own risk. """ +# constant naming for types so pylint: disable=invalid-name try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Any, AnyStr, ByteString, Dict, List, Text - ATTRIBUTE = Dict[Text, Any] # TODO: narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 + ATTRIBUTE = Dict[Text, Any] # narrow this down ITEM = Dict[Text, ATTRIBUTE] RAW_ATTRIBUTE = ITEM NULL = bool # DynamoDB TypeSerializer converts none to {'NULL': True} BOOLEAN = bool - NUMBER = int # TODO: This misses long on Python 2...figure out something for this - STRING = AnyStr # TODO: can be unicode but should not be bytes + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 + NUMBER = int # This misses long on Python 2...figure out something for this + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 + STRING = AnyStr # can be unicode but should not be bytes BINARY = ByteString BINARY_ATTRIBUTE = Dict[Text, BINARY] SET = List # DynamoDB TypeSerializer converts sets into lists diff --git a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py index 13fe0d7b..41058916 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py @@ -32,6 +32,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Callable, Dict, List, Text, Union # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks @@ -64,6 +65,8 @@ def _transform_binary_value(value): def _deserialize_binary(stream): # type: (io.BytesIO) -> Dict[Text, bytes] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a binary object. :param stream: Stream containing serialized object @@ -84,6 +87,8 @@ def _transform_string_value(value): def _deserialize_string(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.STRING] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a string object. :param stream: Stream containing serialized object @@ -106,6 +111,8 @@ def _transform_number_value(value): def _deserialize_number(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.STRING] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a number object. :param stream: Stream containing serialized object @@ -119,6 +126,8 @@ def _deserialize_number(stream): def _deserialize_boolean(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.BOOLEAN] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a boolean object. :param stream: Stream containing serialized object @@ -130,6 +139,8 @@ def _deserialize_boolean(stream): def _deserialize_null(stream): # we want a consistent API but don't use stream, so pylint: disable=unused-argument # type: (io.BytesIO) -> Dict[Text, dynamodb_types.BOOLEAN] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a null object. :param stream: Stream containing serialized object @@ -151,6 +162,8 @@ def _deserialize_set(stream, member_transform): def _deserialize_binary_set(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.SET[dynamodb_types.BINARY]] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a binary set object. :param stream: Stream containing serialized object @@ -161,6 +174,8 @@ def _deserialize_binary_set(stream): def _deserialize_string_set(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.SET[dynamodb_types.STRING]] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a string set object. :param stream: Stream containing serialized object @@ -171,6 +186,8 @@ def _deserialize_string_set(stream): def _deserialize_number_set(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.SET[dynamodb_types.STRING]] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a number set object. :param stream: Stream containing serialized object @@ -181,6 +198,8 @@ def _deserialize_number_set(stream): def _deserialize_list(stream): # type: (io.BytesIO) -> Dict[Text, dynamodb_types.LIST] + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """Deserializes a list object. :param stream: Stream containing serialized object @@ -198,19 +217,20 @@ def _deserialize_map(stream): :type stream: io.BytesIO :rtype: dict """ + # for some reason pylint can't follow the Enum member attributes member_count = decode_length(stream) members = {} # type: dynamodb_types.MAP for _ in range(member_count): key = _deserialize(stream) - if Tag.STRING.dynamodb_tag not in key: + if Tag.STRING.dynamodb_tag not in key: # pylint: disable=no-member raise DeserializationError( 'Malformed serialized map: found "{}" as map key.'.format(list(key.keys())[0]) ) value = _deserialize(stream) - members[key[Tag.STRING.dynamodb_tag]] = value + members[key[Tag.STRING.dynamodb_tag]] = value # pylint: disable=no-member - return {Tag.MAP.dynamodb_tag: members} + return {Tag.MAP.dynamodb_tag: members} # pylint: disable=no-member def _deserialize_function(tag): # type: (bytes) -> Callable @@ -220,17 +240,18 @@ def _deserialize_function(tag): :type tag: dynamodb_encryption_sdk.internal.identifiers.Tag :rtype: callable """ + # for some reason pylint can't follow the Enum member attributes deserialize_functions = { - Tag.BINARY.tag: _deserialize_binary, - Tag.BINARY_SET.tag: _deserialize_binary_set, - Tag.NUMBER.tag: _deserialize_number, - Tag.NUMBER_SET.tag: _deserialize_number_set, - Tag.STRING.tag: _deserialize_string, - Tag.STRING_SET.tag: _deserialize_string_set, - Tag.BOOLEAN.tag: _deserialize_boolean, - Tag.NULL.tag: _deserialize_null, - Tag.LIST.tag: _deserialize_list, - Tag.MAP.tag: _deserialize_map, + Tag.BINARY.tag: _deserialize_binary, # pylint: disable=no-member + Tag.BINARY_SET.tag: _deserialize_binary_set, # pylint: disable=no-member + Tag.NUMBER.tag: _deserialize_number, # pylint: disable=no-member + Tag.NUMBER_SET.tag: _deserialize_number_set, # pylint: disable=no-member + Tag.STRING.tag: _deserialize_string, # pylint: disable=no-member + Tag.STRING_SET.tag: _deserialize_string_set, # pylint: disable=no-member + Tag.BOOLEAN.tag: _deserialize_boolean, # pylint: disable=no-member + Tag.NULL.tag: _deserialize_null, # pylint: disable=no-member + Tag.LIST.tag: _deserialize_list, # pylint: disable=no-member + Tag.MAP.tag: _deserialize_map, # pylint: disable=no-member } try: return deserialize_functions[tag] diff --git a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py index aa0114c8..1f6af4a2 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py @@ -30,6 +30,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Dict, Text # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks @@ -52,7 +53,6 @@ def serialize(material_description): """ material_description_bytes = bytearray(_MATERIAL_DESCRIPTION_VERSION) - # TODO: verify Java sorting order for name, value in sorted(material_description.items(), key=lambda x: x[0]): try: material_description_bytes.extend(encode_value(to_bytes(name))) @@ -62,7 +62,8 @@ def serialize(material_description): 'Invalid name or value in material description: "{name}"="{value}"'.format(name=name, value=value) ) - return {Tag.BINARY.dynamodb_tag: bytes(material_description_bytes)} + # for some reason pylint can't follow the Enum member attributes + return {Tag.BINARY.dynamodb_tag: bytes(material_description_bytes)} # pylint: disable=no-member def deserialize(serialized_material_description): @@ -76,7 +77,10 @@ def deserialize(serialized_material_description): :raises InvalidMaterialDescriptionVersionError: if unknown version is found """ try: - _raw_material_description = serialized_material_description[Tag.BINARY.dynamodb_tag] + # for some reason pylint can't follow the Enum member attributes + _raw_material_description = serialized_material_description[ + Tag.BINARY.dynamodb_tag # pylint: disable=no-member + ] material_description_bytes = io.BytesIO(_raw_material_description) total_bytes = len(_raw_material_description) diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py index cf2cca83..1ca416a1 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py @@ -29,6 +29,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Callable # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks @@ -79,6 +80,8 @@ def _transform_binary_value(value): def _serialize_binary(_attribute): # type: (dynamodb_types.BINARY) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: boto3.dynamodb.types.Binary @@ -104,6 +107,8 @@ def _transform_number_value(value): def _serialize_number(_attribute): # type: (str) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: numbers.Number @@ -124,6 +129,8 @@ def _transform_string_value(value): def _serialize_string(_attribute): # type: (dynamodb_types.STRING) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: six.string_types @@ -134,6 +141,8 @@ def _serialize_string(_attribute): def _serialize_boolean(_attribute): # type: (dynamodb_types.BOOLEAN) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param bool _attribute: Attribute to serialize :returns: Serialized _attribute @@ -144,6 +153,8 @@ def _serialize_boolean(_attribute): def _serialize_null(_attribute): # type: (dynamodb_types.NULL) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param _attribute: Attribute to serialize :type _attribute: types.NoneType @@ -203,6 +214,8 @@ def _serialize_string_set(_attribute): def _serialize_list(_attribute): # type: (dynamodb_types.LIST) -> bytes + # pylint: disable=no-member + # for some reason pylint can't follow the Enum member attributes """ :param list _attribute: Attribute to serialize :returns: Serialized _attribute @@ -226,7 +239,8 @@ def _serialize_map(_attribute): """ serialized_attribute = io.BytesIO() serialized_attribute.write(_RESERVED) - serialized_attribute.write(Tag.MAP.tag) + # for some reason pylint can't follow the Enum member attributes + serialized_attribute.write(Tag.MAP.tag) # pylint: disable=no-member serialized_attribute.write(encode_length(_attribute)) sorted_items = _sorted_key_map(item=_attribute, transform=_transform_string_value) @@ -240,17 +254,18 @@ def _serialize_map(_attribute): def _serialize_function(dynamodb_tag): # type: (str) -> Callable[[dynamodb_types.ATTRIBUTE], bytes] """Locates the appropriate serialization function for the specified DynamoDB attribute tag.""" + # for some reason pylint can't follow the Enum member attributes serialize_functions = { - Tag.BINARY.dynamodb_tag: _serialize_binary, - Tag.BINARY_SET.dynamodb_tag: _serialize_binary_set, - Tag.NUMBER.dynamodb_tag: _serialize_number, - Tag.NUMBER_SET.dynamodb_tag: _serialize_number_set, - Tag.STRING.dynamodb_tag: _serialize_string, - Tag.STRING_SET.dynamodb_tag: _serialize_string_set, - Tag.BOOLEAN.dynamodb_tag: _serialize_boolean, - Tag.NULL.dynamodb_tag: _serialize_null, - Tag.LIST.dynamodb_tag: _serialize_list, - Tag.MAP.dynamodb_tag: _serialize_map, + Tag.BINARY.dynamodb_tag: _serialize_binary, # pylint: disable=no-member + Tag.BINARY_SET.dynamodb_tag: _serialize_binary_set, # pylint: disable=no-member + Tag.NUMBER.dynamodb_tag: _serialize_number, # pylint: disable=no-member + Tag.NUMBER_SET.dynamodb_tag: _serialize_number_set, # pylint: disable=no-member + Tag.STRING.dynamodb_tag: _serialize_string, # pylint: disable=no-member + Tag.STRING_SET.dynamodb_tag: _serialize_string_set, # pylint: disable=no-member + Tag.BOOLEAN.dynamodb_tag: _serialize_boolean, # pylint: disable=no-member + Tag.NULL.dynamodb_tag: _serialize_null, # pylint: disable=no-member + Tag.LIST.dynamodb_tag: _serialize_list, # pylint: disable=no-member + Tag.MAP.dynamodb_tag: _serialize_map, # pylint: disable=no-member } try: return serialize_functions[dynamodb_tag] diff --git a/src/dynamodb_encryption_sdk/internal/identifiers.py b/src/dynamodb_encryption_sdk/internal/identifiers.py index b0474ec8..94d7bd41 100644 --- a/src/dynamodb_encryption_sdk/internal/identifiers.py +++ b/src/dynamodb_encryption_sdk/internal/identifiers.py @@ -73,7 +73,7 @@ def __init__(self, tag, dynamodb_tag, element_tag=None): """Sets up new Tag object. :param bytes tag: DynamoDB Encryption SDK tag - :param bytes dynamodb_tag: DynamoDB tag + :param str dynamodb_tag: DynamoDB tag :param bytes element_tag: The type of tag contained within attributes of this type """ self.tag = tag diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index ae8ebe16..988576b2 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -174,7 +174,8 @@ def _item_transformer(crypto_transformer): def decrypt_list_of_items(crypto_config, decrypt_method, items): # type: (CryptoConfig, Callable, Iterable[Any]) -> Iterable[Any] - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Iterate through a list of encrypted items, decrypting each item and yielding the plaintext item. :param CryptoConfig crypto_config: :class:`CryptoConfig` to use @@ -190,7 +191,8 @@ def decrypt_list_of_items(crypto_config, decrypt_method, items): def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently decrypt multiple items after getting them from the table with a scan or query method. :param callable decrypt_method: Method to use to decrypt items @@ -211,7 +213,8 @@ def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwarg def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently decrypt an item after getting it from the table. :param callable decrypt_method: Method to use to decrypt item @@ -234,7 +237,8 @@ def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently decrypt multiple items after getting them in a batch request. :param callable decrypt_method: Method to use to decrypt items @@ -265,7 +269,8 @@ def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, ** def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently encrypt an item before putting it to the table. :param callable encrypt_method: Method to use to encrypt items @@ -285,7 +290,8 @@ def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwarg def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict - # TODO: narrow this down + # narrow this down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Transparently encrypt multiple items before putting them in a batch request. :param callable encrypt_method: Method to use to encrypt items diff --git a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py index 7fa47854..ea7a55f2 100644 --- a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py +++ b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py @@ -37,8 +37,9 @@ from . import CryptographicMaterialsProvider try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import from typing import Dict, Optional, Text, Tuple # noqa pylint: disable=unused-import + + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass @@ -137,6 +138,7 @@ def from_material_description(cls, material_description, description_key, defaul @attr.s(init=False) class AwsKmsCryptographicMaterialsProvider(CryptographicMaterialsProvider): + # pylint: disable=too-many-instance-attributes """Cryptographic materials provider for use with the AWS Key Management Service (KMS). .. note:: diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index 80892882..ce074301 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -194,7 +194,7 @@ def _ttl_action(self): if time_since_updated < self._version_ttl: return TtlActions.LIVE - elif time_since_updated < self._version_ttl + _GRACE_PERIOD: + if time_since_updated < self._version_ttl + _GRACE_PERIOD: return TtlActions.GRACE_PERIOD _LOGGER.debug("TTL Expired because known version has expired") @@ -255,12 +255,13 @@ def _get_most_recent_version(self, allow_local): provider = self._cache.get(max_version) except KeyError: provider = self._get_provider(max_version) - received_version = self._provider_store.version_from_material_description(provider._material_description) - # TODO: ^ should we promote material description from hidden? + received_version = self._provider_store.version_from_material_description( + provider._material_description # pylint: disable=protected-access + ) _LOGGER.debug("Caching materials provider version %d", received_version) - self._version = received_version - self._last_updated = time.time() + self._version = received_version # pylint: disable=attribute-defined-outside-init + self._last_updated = time.time() # pylint: disable=attribute-defined-outside-init self._cache.put(received_version, provider) finally: self._lock.release() diff --git a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py index d817976c..e03b57b9 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/__init__.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/__init__.py @@ -21,7 +21,7 @@ ) try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Text, Optional # noqa pylint: disable=unused-import + from typing import Dict, Optional, Text # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/src/dynamodb_encryption_sdk/material_providers/store/meta.py b/src/dynamodb_encryption_sdk/material_providers/store/meta.py index c16cd5e7..567c57be 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/meta.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/meta.py @@ -121,7 +121,7 @@ def create_table(cls, client, table_name, read_units, write_units): ProvisionedThroughput={"ReadCapacityUnits": read_units, "WriteCapacityUnits": write_units}, ) except botocore.exceptions.ClientError: - raise Exception("TODO: Could not create table") + raise Exception("Could not create table") def _load_materials(self, material_name, version): # type: (Text, int) -> Tuple[JceNameLocalDelegatedKey, JceNameLocalDelegatedKey] @@ -152,9 +152,10 @@ def _load_materials(self, material_name, version): key_encoding=KeyEncodingType.RAW, ) except KeyError: - raise Exception("TODO: Invalid record") + raise Exception("Invalid record") - # TODO: handle if the material type version is not in the item + # need to handle if the material type version is not in the item + # https://github.com/aws/aws-dynamodb-encryption-python/issues/140 if item[MetaStoreAttributeNames.MATERIAL_TYPE_VERSION.value] != MetaStoreValues.MATERIAL_TYPE_VERSION.value: raise InvalidVersionError( 'Unsupported material type: "{}"'.format(item[MetaStoreAttributeNames.MATERIAL_TYPE_VERSION.value]) @@ -301,12 +302,12 @@ def version_from_material_description(self, material_description): try: info = material_description[_MATERIAL_DESCRIPTION_META_FIELD] except KeyError: - raise Exception("TODO: No info found") + raise Exception("No info found") try: return int(info.split("#", 1)[1]) except (IndexError, ValueError): - raise Exception("TODO: Malformed info") + raise Exception("Malformed info") def max_version(self, material_name): # (Text) -> int diff --git a/src/dynamodb_encryption_sdk/materials/__init__.py b/src/dynamodb_encryption_sdk/materials/__init__.py index ba07d0c8..66797ec6 100644 --- a/src/dynamodb_encryption_sdk/materials/__init__.py +++ b/src/dynamodb_encryption_sdk/materials/__init__.py @@ -19,6 +19,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Dict, Text # noqa pylint: disable=unused-import + from mypy_extensions import NoReturn # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks diff --git a/src/dynamodb_encryption_sdk/materials/wrapped.py b/src/dynamodb_encryption_sdk/materials/wrapped.py index daf8638a..f85924ea 100644 --- a/src/dynamodb_encryption_sdk/materials/wrapped.py +++ b/src/dynamodb_encryption_sdk/materials/wrapped.py @@ -101,9 +101,10 @@ def __attrs_post_init__(self): self._content_key_from_material_description() ) # noqa pylint: disable=attribute-defined-outside-init else: - self._content_key, self._material_description = ( - self._generate_content_key() - ) # noqa pylint: disable=attribute-defined-outside-init + ( + self._content_key, + self._material_description, + ) = self._generate_content_key() # noqa pylint: disable=attribute-defined-outside-init @staticmethod def _wrapping_transformation(algorithm): diff --git a/src/dynamodb_encryption_sdk/structures.py b/src/dynamodb_encryption_sdk/structures.py index 0dd35461..61b329c6 100644 --- a/src/dynamodb_encryption_sdk/structures.py +++ b/src/dynamodb_encryption_sdk/structures.py @@ -223,7 +223,7 @@ class TableIndex(object): partition = attr.ib(validator=attr.validators.instance_of(six.string_types)) sort = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(six.string_types)), default=None) - def __init__(self, partition, sort=None): + def __init__(self, partition, sort=None): # noqa=D107 # type: (Text, Optional[Text]) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 diff --git a/src/dynamodb_encryption_sdk/transform.py b/src/dynamodb_encryption_sdk/transform.py index 76b22c6c..347024d3 100644 --- a/src/dynamodb_encryption_sdk/transform.py +++ b/src/dynamodb_encryption_sdk/transform.py @@ -24,7 +24,8 @@ def dict_to_ddb(item): # type: (Dict[str, Any]) -> Dict[str, Any] - # TODO: narrow these types down + # narrow these types down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Converts a native Python dictionary to a raw DynamoDB item. :param dict item: Native item @@ -37,7 +38,8 @@ def dict_to_ddb(item): def ddb_to_dict(item): # type: (Dict[str, Any]) -> Dict[str, Any] - # TODO: narrow these types down + # narrow these types down + # https://github.com/aws/aws-dynamodb-encryption-python/issues/66 """Converts a raw DynamoDB item to a native Python dictionary. :param dict item: DynamoDB item diff --git a/src/pylintrc b/src/pylintrc index bd252833..bc0406f6 100644 --- a/src/pylintrc +++ b/src/pylintrc @@ -3,8 +3,11 @@ disable = bad-continuation, # we let black handle this ungrouped-imports, # we let isort handle this - useless-object-inheritance, # we need to support Python 2, so no, not useless duplicate-code, # causes lots of problems with implementations of common interfaces + # All below are disabled because we need to support Python 2 + useless-object-inheritance, + raise-missing-from, + super-with-arguments, [BASIC] # Allow function names up to 50 characters diff --git a/test/acceptance/acceptance_test_utils.py b/test/acceptance/acceptance_test_utils.py index 5d66bc65..482d182a 100644 --- a/test/acceptance/acceptance_test_utils.py +++ b/test/acceptance/acceptance_test_utils.py @@ -159,12 +159,12 @@ def _meta_table_prep(table_name, items_filename): table_data = json.load(f) request_items = {} - for table_name, items in table_data.items(): + for this_table_name, items in table_data.items(): requests = [] for item in items: _decode_item(item) requests.append({"PutRequest": {"Item": item}}) - request_items[table_name] = requests + request_items[this_table_name] = requests client.batch_write_item(RequestItems=request_items) diff --git a/test/functional/delegated_keys/test_jce.py b/test/functional/delegated_keys/test_jce.py index ba880399..08f5446e 100644 --- a/test/functional/delegated_keys/test_jce.py +++ b/test/functional/delegated_keys/test_jce.py @@ -78,7 +78,7 @@ def build_short_key_cases(): @pytest.mark.parametrize("algorithm, key_bits, too_short, error_message", build_short_key_cases()) def test_warn_on_short_keys(caplog, algorithm, key_bits, too_short, error_message): with caplog.at_level(logging.DEBUG): - _test = JceNameLocalDelegatedKey.generate(algorithm, key_bits) + _test = JceNameLocalDelegatedKey.generate(algorithm, key_bits) # noqa=F401 logging_results = caplog.text assert (too_short and error_message in logging_results) or (not too_short and error_message not in logging_results) diff --git a/test/functional/encrypted/test_client.py b/test/functional/encrypted/test_client.py index 83f0f250..40c055df 100644 --- a/test/functional/encrypted/test_client.py +++ b/test/functional/encrypted/test_client.py @@ -14,8 +14,10 @@ import hypothesis import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import ( + TEST_REGION_NAME, TEST_TABLE_NAME, build_static_jce_cmp, client_batch_items_unprocessed_check, @@ -39,25 +41,25 @@ def pytest_generate_tests(metafunc): def _client_cycle_single_item_check(materials_provider, initial_actions, initial_item): return client_cycle_single_item_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) def _client_cycle_batch_items_check(materials_provider, initial_actions, initial_item): return client_cycle_batch_items_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) def _client_cycle_batch_items_check_scan_paginator(materials_provider, initial_actions, initial_item): return client_cycle_batch_items_check_scan_paginator( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) def _client_batch_items_unprocessed_check(materials_provider, initial_actions, initial_item): client_batch_items_unprocessed_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) diff --git a/test/functional/encrypted/test_resource.py b/test/functional/encrypted/test_resource.py index 3c6a01e2..95ed9d28 100644 --- a/test/functional/encrypted/test_resource.py +++ b/test/functional/encrypted/test_resource.py @@ -13,8 +13,10 @@ """Functional tests for ``dynamodb_encryption_sdk.encrypted.resource``.""" import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import ( + TEST_REGION_NAME, TEST_TABLE_NAME, build_static_jce_cmp, resource_batch_items_unprocessed_check, @@ -34,12 +36,14 @@ def pytest_generate_tests(metafunc): def _resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item): - resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2") + resource_cycle_batch_items_check( + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME + ) def _resource_batch_items_unprocessed_check(materials_provider, initial_actions, initial_item): resource_batch_items_unprocessed_check( - materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME ) diff --git a/test/functional/encrypted/test_table.py b/test/functional/encrypted/test_table.py index fe9339ad..736edb3b 100644 --- a/test/functional/encrypted/test_table.py +++ b/test/functional/encrypted/test_table.py @@ -14,8 +14,10 @@ import hypothesis import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import from ..functional_test_utils import ( + TEST_REGION_NAME, TEST_TABLE_NAME, build_static_jce_cmp, set_parametrized_actions, @@ -37,7 +39,7 @@ def pytest_generate_tests(metafunc): def _table_cycle_check(materials_provider, initial_actions, initial_item): - return table_cycle_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2") + return table_cycle_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, TEST_REGION_NAME) def test_ephemeral_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): @@ -47,14 +49,16 @@ def test_ephemeral_item_cycle(example_table, some_cmps, parametrized_actions, pa def test_ephemeral_item_cycle_batch_writer(example_table, some_cmps, parametrized_actions, parametrized_item): """Test a small number of curated CMPs against a small number of curated items.""" - table_cycle_batch_writer_check(some_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2") + table_cycle_batch_writer_check( + some_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, TEST_REGION_NAME + ) def test_batch_writer_unprocessed(example_table, parametrized_actions, parametrized_item): """Test Unprocessed Items handling with a single ephemeral static CMP against a small number of curated items.""" cmp = build_static_jce_cmp("AES", 256, "HmacSHA256", 256) table_batch_writer_unprocessed_items_check( - cmp, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2" + cmp, parametrized_actions, parametrized_item, TEST_TABLE_NAME, TEST_REGION_NAME ) @@ -67,7 +71,9 @@ def test_ephemeral_item_cycle_slow(example_table, all_the_cmps, parametrized_act @pytest.mark.slow def test_ephemeral_item_cycle_batch_writer_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): """Test a small number of curated CMPs against a small number of curated items.""" - table_cycle_batch_writer_check(all_the_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2") + table_cycle_batch_writer_check( + all_the_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, TEST_REGION_NAME + ) @pytest.mark.travis_isolation diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 950aee47..8902c626 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -78,11 +78,16 @@ TEST_BATCH_KEYS = [{name: value["value"] for name, value in key.items()} for key in TEST_BATCH_INDEXES] +@pytest.fixture(scope="module") +def mock_ddb_service(): + """Centralize service mock to avoid resetting service for tests that use multiple tables.""" + with mock_dynamodb2(): + yield boto3.client("dynamodb", region_name=TEST_REGION_NAME) + + @pytest.fixture -def example_table(): - mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) - ddb.create_table( +def example_table(mock_ddb_service): + mock_ddb_service.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ {"AttributeName": "partition_attribute", "KeyType": "HASH"}, @@ -93,16 +98,13 @@ def example_table(): ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) - yield - ddb.delete_table(TableName=TEST_TABLE_NAME) - mock_dynamodb2().stop() + yield mock_ddb_service + mock_ddb_service.delete_table(TableName=TEST_TABLE_NAME) @pytest.fixture -def table_with_local_secondary_indexes(): - mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) - ddb.create_table( +def table_with_local_secondary_indexes(mock_ddb_service): + mock_ddb_service.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ {"AttributeName": "partition_attribute", "KeyType": "HASH"}, @@ -126,16 +128,13 @@ def table_with_local_secondary_indexes(): ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) - yield - ddb.delete_table(TableName=TEST_TABLE_NAME) - mock_dynamodb2().stop() + yield mock_ddb_service + mock_ddb_service.delete_table(TableName=TEST_TABLE_NAME) @pytest.fixture -def table_with_global_secondary_indexes(): - mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) - ddb.create_table( +def table_with_global_secondary_indexes(mock_ddb_service): + mock_ddb_service.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ {"AttributeName": "partition_attribute", "KeyType": "HASH"}, @@ -161,9 +160,8 @@ def table_with_global_secondary_indexes(): ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) - yield - ddb.delete_table(TableName=TEST_TABLE_NAME) - mock_dynamodb2().stop() + yield mock_ddb_service + mock_ddb_service.delete_table(TableName=TEST_TABLE_NAME) class PassThroughCryptographicMaterialsProviderThatRequiresAttributes(CryptographicMaterialsProvider): @@ -178,7 +176,8 @@ class PassThroughCryptographicMaterialsProviderThatRequiresAttributes(Cryptograp def __init__(self, passthrough_cmp): self._passthrough_cmp = passthrough_cmp - def _assert_attributes_set(self, encryption_context): + @staticmethod + def _assert_attributes_set(encryption_context): # type: (EncryptionContext) -> None if not encryption_context.attributes: raise ValueError("Encryption context attributes MUST be set!") @@ -386,7 +385,7 @@ def diverse_item(): return copy.deepcopy(base_item) -_reserved_attributes = set([attr.value for attr in ReservedAttributes]) +_reserved_attributes = {attr.value for attr in ReservedAttributes} def return_requestitems_as_unprocessed(*args, **kwargs): @@ -520,9 +519,7 @@ def cycle_batch_item_check( def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initial_item): - """Check that cycling (plaintext->encrypted->decrypted) items with the Table batch writer - has the expected results. - """ + """Cycling (plaintext->encrypted->decrypted) items with the Table batch writer should have the expected results.""" check_attribute_actions = initial_actions.copy() check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) items = _generate_items(initial_item, _nop_transformer) @@ -531,7 +528,7 @@ def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initia for item in items: writer.put_item(item) - ddb_keys = [key for key in TEST_BATCH_KEYS] + ddb_keys = copy.copy(TEST_BATCH_KEYS) encrypted_items = [raw_table.get_item(Key=key, ConsistentRead=True)["Item"] for key in ddb_keys] check_many_encrypted_items( actual=encrypted_items, expected=items, attribute_actions=check_attribute_actions, transformer=_nop_transformer @@ -757,7 +754,8 @@ def client_cycle_batch_items_check_scan_paginator( scan the table with raw client paginator to get encrypted items, scan the table with encrypted client paginator to get decrypted items, then verify that all items appear to have been encrypted correctly. - """ + """ # noqa=D401 + # pylint: disable=too-many-locals kwargs = {} if region_name is not None: kwargs["region_name"] = region_name diff --git a/test/functional/hypothesis_strategies.py b/test/functional/hypothesis_strategies.py index 3550a411..6a39d4cf 100644 --- a/test/functional/hypothesis_strategies.py +++ b/test/functional/hypothesis_strategies.py @@ -22,7 +22,6 @@ suppress_health_check=( hypothesis.HealthCheck.too_slow, hypothesis.HealthCheck.data_too_large, - hypothesis.HealthCheck.hung_test, hypothesis.HealthCheck.large_base_example, ), deadline=None, @@ -32,9 +31,7 @@ # _MIN_NUMBER = Decimal('1E-128') # The DDB min is 1E-130, but DYNAMODB_CONTEXT Emin is -128 # _MAX_NUMBER = Decimal('9.9999999999999999999999999999999999999E+125') -# TODO: I would like to test the full range of possible number values, but boto3 does not -# correctly handle conversion of large edge case values at this time. We will work to fix -# that, but in the meantime, we will just use the happy path numbers. +# boto3 does not correctly handle conversion of large edge case values at this time _MIN_NUMBER = Decimal("1E-38") _MAX_NUMBER = Decimal("9.{}E37".format("9" * 37)) @@ -68,7 +65,8 @@ def _negative(val): ddb_set_types = ddb_string_set | ddb_number_set | ddb_binary_set ddb_attribute_names = text(min_size=1, max_size=255) -# TODO: List and Map types have a max depth of 32 +# List and Map types have a max depth of 32 +# https://github.com/aws/aws-dynamodb-encryption-python/issues/141 ddb_map_type = deferred( lambda: dictionaries( keys=ddb_attribute_names, values=(ddb_scalar_types | ddb_set_types | ddb_list_type | ddb_map_type), min_size=1 diff --git a/test/functional/material_providers/store/test_meta.py b/test/functional/material_providers/store/test_meta.py index 908fc9dc..57d5611a 100644 --- a/test/functional/material_providers/store/test_meta.py +++ b/test/functional/material_providers/store/test_meta.py @@ -11,18 +11,13 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Functional tests for ``dynamodb_encryption_sdk.material_providers.store.meta``.""" -import base64 -import os - -import boto3 import pytest -from moto import mock_dynamodb2 from dynamodb_encryption_sdk.exceptions import NoKnownVersionError from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore, MetaStoreAttributeNames from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider -from ...functional_test_utils import build_static_jce_cmp, mock_metastore +from ...functional_test_utils import mock_metastore # noqa=F401 pylint: disable=unused-import pytestmark = [pytest.mark.functional, pytest.mark.local] diff --git a/test/functional/material_providers/test_most_recent.py b/test/functional/material_providers/test_most_recent.py index 1d41ef83..967b1da5 100644 --- a/test/functional/material_providers/test_most_recent.py +++ b/test/functional/material_providers/test_most_recent.py @@ -21,12 +21,10 @@ from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider from dynamodb_encryption_sdk.material_providers.store import ProviderStore -from ..functional_test_utils import ( # pylint: disable=unused-import - TEST_TABLE_NAME, - check_metastore_cache_use_encrypt, - example_table, - mock_metastore, -) +from ..functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import mock_metastore # noqa=F401 pylint: disable=unused-import +from ..functional_test_utils import TEST_TABLE_NAME, check_metastore_cache_use_encrypt pytestmark = [pytest.mark.functional, pytest.mark.local] diff --git a/test/functional/materials/test_raw.py b/test/functional/materials/test_raw.py index f093f811..65cb0c62 100644 --- a/test/functional/materials/test_raw.py +++ b/test/functional/materials/test_raw.py @@ -24,7 +24,7 @@ def test_no_encryption_key(): encryption_materials = RawEncryptionMaterials(signing_key=signing_key) with pytest.raises(AttributeError) as excinfo: - encryption_materials.encryption_key + encryption_materials.encryption_key # calls a property, so pylint: disable=pointless-statement excinfo.match("No encryption key available") @@ -34,6 +34,6 @@ def test_no_decryption_key(): decryption_materials = RawDecryptionMaterials(verification_key=verification_key) with pytest.raises(AttributeError) as excinfo: - decryption_materials.decryption_key + decryption_materials.decryption_key # calls a property, so pylint: disable=pointless-statement excinfo.match("No decryption key available") diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py index 0c180fe9..aec90179 100644 --- a/test/functional/test_structures.py +++ b/test/functional/test_structures.py @@ -18,34 +18,31 @@ from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.structures import AttributeActions, TableIndex, TableInfo -from .functional_test_utils import ( - TEST_TABLE_NAME, - example_table, - table_with_global_secondary_indexes, - table_with_local_secondary_indexes, -) +from .functional_test_utils import example_table # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import mock_ddb_service # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import table_with_global_secondary_indexes # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import table_with_local_secondary_indexes # noqa=F401 pylint: disable=unused-import +from .functional_test_utils import TEST_REGION_NAME, TEST_TABLE_NAME pytestmark = [pytest.mark.functional, pytest.mark.local] -# TODO: There is a way to parameterize fixtures, but the existing docs on that are very unclear. -# This will get us what we need for now, but we should come back to this to clean this up later. def test_tableinfo_refresh_indexes_no_secondary_indexes(example_table): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) table = TableInfo(name=TEST_TABLE_NAME) table.refresh_indexed_attributes(client) def test_tableinfo_refresh_indexes_with_gsis(table_with_global_secondary_indexes): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) table = TableInfo(name=TEST_TABLE_NAME) table.refresh_indexed_attributes(client) def test_tableinfo_refresh_indexes_with_lsis(table_with_local_secondary_indexes): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) table = TableInfo(name=TEST_TABLE_NAME) table.refresh_indexed_attributes(client) diff --git a/test/integration/integration_test_utils.py b/test/integration/integration_test_utils.py index 7eb3ea7d..71106ef7 100644 --- a/test/integration/integration_test_utils.py +++ b/test/integration/integration_test_utils.py @@ -20,7 +20,8 @@ # convenience imports try: - from ..functional import functional_test_utils, hypothesis_strategies + from ..functional import hypothesis_strategies # noqa=F401 pylint: disable=unused-import + from ..functional import functional_test_utils except (ImportError, ValueError, SystemError): if "AWS_ENCRYPTION_SDK_EXAMPLES_TESTING" not in os.environ: raise @@ -53,8 +54,8 @@ def _build_kms_cmp(require_attributes): inner_cmp = AwsKmsCryptographicMaterialsProvider(key_id=cmk_arn_value()) if require_attributes: return functional_test_utils.PassThroughCryptographicMaterialsProviderThatRequiresAttributes(inner_cmp) - else: - return inner_cmp + + return inner_cmp def set_parameterized_kms_cmps(metafunc, require_attributes=True): diff --git a/test/integration/material_providers/store/test_meta.py b/test/integration/material_providers/store/test_meta.py index c092ba21..7ee28deb 100644 --- a/test/integration/material_providers/store/test_meta.py +++ b/test/integration/material_providers/store/test_meta.py @@ -16,7 +16,7 @@ from dynamodb_encryption_sdk.exceptions import NoKnownVersionError from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore, MetaStoreAttributeNames -from ...integration_test_utils import temp_metastore # pylint: disable=unused-import +from ...integration_test_utils import temp_metastore # noqa=F401 pylint: disable=unused-import pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] diff --git a/test/integration/material_providers/test_most_recent.py b/test/integration/material_providers/test_most_recent.py index 90c2ccf8..ba843cac 100644 --- a/test/integration/material_providers/test_most_recent.py +++ b/test/integration/material_providers/test_most_recent.py @@ -13,11 +13,9 @@ """Load testing using MostRecentProvider and MetaStore.""" import pytest -from ..integration_test_utils import ( # pylint: disable=unused-import - ddb_table_name, - functional_test_utils, - temp_metastore, -) +from ..integration_test_utils import ddb_table_name # noqa=F401 pylint: disable=unused-import +from ..integration_test_utils import temp_metastore # noqa=F401 pylint: disable=unused-import +from ..integration_test_utils import functional_test_utils pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] diff --git a/test/pylintrc b/test/pylintrc index 123c1870..ce2bba60 100644 --- a/test/pylintrc +++ b/test/pylintrc @@ -6,11 +6,14 @@ disable = bad-continuation, # we let black handle this ungrouped-imports, # we let isort handle this no-member, # raised on patched objects with mock checks - useless-object-inheritance, # we need to support Python 2, so no, not useless duplicate-code, # unit tests for similar things tend to be similar protected-access, # raised when calling _ methods redefined-outer-name, # raised when using pytest-mock unused-argument, # raised when patches and fixtures are needed but not called + # All below are disabled because we need to support Python 2 + useless-object-inheritance, + raise-missing-from, + super-with-arguments, [DESIGN] max-args = 10 diff --git a/test/requirements.txt b/test/requirements.txt index 652ce531..2d87d831 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,5 @@ -hypothesis>=3.63.0 +hypothesis>=5.0.0;python_version>='3' +hypothesis==4.57.1;python_version=='2.7' mock moto>=1.3.8 pytest>=3.4.0 diff --git a/test/source-build-check.sh b/test/source-build-check.sh index 91403af4..22e31a83 100755 --- a/test/source-build-check.sh +++ b/test/source-build-check.sh @@ -4,20 +4,26 @@ #NOTE: Humans should not run this file directly. If you want to run this check, use the # tox to run the "sourcebuildcheck" test environment. -WORKINGDIR=$1 -DISTDIR=$2 +WORKINGDIR=${1} +DISTDIR=${2} + +echo "Performing source build check" +echo "Using working directory ${WORKINGDIR}" +echo "Using dist directory ${DISTDIR}" echo "Locating the source build and copying it into the working directory." -DISTFILE=`ls $DISTDIR/dynamodb-encryption-sdk-*.tar.gz | tail -1` -cp $DISTFILE $WORKINGDIR -DISTFILE=`ls $WORKINGDIR/dynamodb-encryption-sdk-*.tar.gz | tail -1` +DISTFILE=$(ls ${DISTDIR}/dynamodb-encryption-sdk-*.tar.gz | tail -1) +echo "Found source build at ${DISTFILE}" +cp ${DISTFILE} ${WORKINGDIR} echo "Extracting the source build." -cd $WORKINGDIR -tar xzvf $DISTFILE -rm $DISTFILE -EXTRACTEDDIR=`ls | tail -1` -cd $EXTRACTEDDIR +cd ${WORKINGDIR} +NEWDISTFILE=$(ls dynamodb-encryption-sdk-*.tar.gz | tail -1) +echo "Using distfile ${NEWDISTFILE}" +tar xzvf ${NEWDISTFILE} +rm ${NEWDISTFILE} +EXTRACTEDDIR=$(ls | tail -1) +cd ${EXTRACTEDDIR} echo "Installing requirements from extracted source build." pip install -r test/requirements.txt diff --git a/test/unit/internal/formatting/test_attribute.py b/test/unit/internal/formatting/test_attribute.py index 71235e0c..5fc0617f 100644 --- a/test/unit/internal/formatting/test_attribute.py +++ b/test/unit/internal/formatting/test_attribute.py @@ -10,8 +10,11 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit tests for ``dynamodb_encryption_sdk.internal.formatting.serialize.attribute`` -and ``dynamodb_encryption_sdk.internal.formatting.deserialize.attribute``.""" +"""De/serialization unit tests. + +Tests for ``dynamodb_encryption_sdk.internal.formatting.serialize.attribute`` +and ``dynamodb_encryption_sdk.internal.formatting.deserialize.attribute``. +""" import pytest from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import _sorted_key_map diff --git a/test/unit/material_providers/test_aws_kms.py b/test/unit/material_providers/test_aws_kms.py index efedfd90..edcd301d 100644 --- a/test/unit/material_providers/test_aws_kms.py +++ b/test/unit/material_providers/test_aws_kms.py @@ -316,7 +316,6 @@ def test_select_id(default_kms_cmp): assert test is default_kms_cmp._key_id -# TODO: vectorize @pytest.mark.parametrize( "attribute, expected_value", ( @@ -338,7 +337,6 @@ def test_attribute_to_value_wrong_type(default_kms_cmp): excinfo.match(r"Attribute of type *") -# TODO: vectorize @pytest.mark.parametrize( "encryption_context, additional_expected_keypairs", ( @@ -413,7 +411,6 @@ def test_decrypt_initial_material_fail(default_kms_cmp, patch_kms_client): excinfo.match("Failed to unwrap AWS KMS protected materials") -# TODO: vectorize @pytest.mark.parametrize( "description, method_name, key_name", (("AES/256", "_encryption_key", "encryption_key"), ("HmacSHA256/256", "_mac_key", "mac_key")), diff --git a/tox.ini b/tox.ini index 0305eb5f..6fe21863 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{27,34,35,36,37}-{local,integ,ddb,examples}-fast, + py{27,35,36,37,38}-{local,integ,ddb,examples}-fast, nocmk, sourcebuildcheck, docs, bandit, doc8, readme, flake8{,-tests,-examples}, pylint{,-tests,-examples}, @@ -52,6 +52,8 @@ passenv = AWS_DEFAULT_REGION sitepackages = False deps = -rtest/requirements.txt +# 'download' forces tox to always upgrade pip to the latest +download = true commands = # Only run small test scenario sets local-fast: {[testenv:base-command]commands} test/ -m "local and not slow and not veryslow and not nope" @@ -86,6 +88,11 @@ commands = [testenv:nocmk] basepython = python3 sitepackages = False +######################################################### +# Do not pass through or set any environment variables! # +passenv = +setenv = +######################################################### deps = -rtest/requirements.txt commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" @@ -167,7 +174,7 @@ commands = basepython = python3 deps = coverage - mypy<=0.560 + mypy mypy_extensions typing>=3.6.2 @@ -215,8 +222,10 @@ deps = {[testenv:flake8]deps} commands = flake8 \ # Ignore F811 redefinition errors in tests (breaks with pytest-mock use) - # Ignore D103 docstring requirements for tests - --ignore F811,D103 \ + # Ignore D101-107 docstring requirements for tests + # E203 is not PEP8 compliant https://github.com/ambv/black#slices + # W503 is not PEP8 compliant https://github.com/ambv/black#line-breaks--binary-operators + --ignore F811,D101,D102,D103,D107,E203,W503 \ test/ [testenv:flake8-examples] @@ -301,14 +310,15 @@ commands = seed-isort-config [testenv:isort] basepython = python3 -deps = isort +# We need >=5.0.0 because +# several configuration settings changed with 5.0.0 +deps = isort>=5.0.0 commands = isort \ - -rc \ - src \ - test \ - examples/ \ - doc \ - setup.py \ + src \ + test \ + examples/ \ + doc \ + setup.py \ --skip examples/test/examples_test_utils.py \ {posargs}