From eecfefb8ecae9ac47bb6e9f3fc6228126a4d72d3 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 12:00:42 +0800 Subject: [PATCH 01/11] update travis to test py3 --- .gitignore | 2 ++ .travis.yml | 10 ++++++---- requirements.txt | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index cb86ac3..5422f7f 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,5 @@ target/ *~ tests/zappa_settings.json tests/example_template_outputs +.venv +Pipfile* diff --git a/.travis.yml b/.travis.yml index f1c2ea2..479d797 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,18 @@ language: python python: - - "2.7" -# - "3.5" +# - "2.7" + - "3.5" + - "3.6" + - "3.7" # command to install dependencies cache: - pip install: - - "pip install setuptools --upgrade; pip install -r test_requirements.txt; python setup.py install" + - "pip install setuptools --upgrade; pip install -r test_requirements.txt; python setup.py install" # command to run tests env: - TESTCASE=tests/tests.py -script: +script: - nosetests $TESTCASE --with-coverage --cover-package=nodb --with-timer # - coverage combine --append after_success: diff --git a/requirements.txt b/requirements.txt index ab1bb72..690ec68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ jmespath>=0.9.2 packaging>=16.8 pbr>=2.0.0 pyparsing>=2.2.0 -python-dateutil==2.6.0 +python-dateutil>=2.6.0 s3transfer>=0.1.10 six>=1.10.0 From f274cd42ca59ba5092650922c0beb7a9514d5a40 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 12:16:31 +0800 Subject: [PATCH 02/11] test_requirements.txt greaterthan versions --- test_requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_requirements.txt b/test_requirements.txt index 69740ab..4a94c17 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,5 +1,5 @@ -coverage==4.3.4 +coverage>=4.3.4 coveralls>=1.1 -moto==1.3.7 -nose==1.3.7 +moto>=1.3.7 +nose>=1.3.7 nose-timer>=0.6.0 From b4e853b4398fe0088c09e1833070e3ed163ef228 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 12:25:37 +0800 Subject: [PATCH 03/11] remove dependencies from requirements.txt --- requirements.txt | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 690ec68..10b770f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,6 @@ -appdirs>=1.4.3 boto3>=1.4.4 -botocore>=1.5.38 -docutils>=0.13.1 +# @bendog not sure why the following are needed +appdirs>=1.4.3 funcsigs>=1.0.2 -futures>=3.0.5 -jmespath>=0.9.2 packaging>=16.8 pbr>=2.0.0 -pyparsing>=2.2.0 -python-dateutil>=2.6.0 -s3transfer>=0.1.10 -six>=1.10.0 From e0fb0135c3b41ae9ffd27575bd3c8c4d04745822 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 12:34:10 +0800 Subject: [PATCH 04/11] added python 2.7 back to travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7925698..de45385 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: -# - "2.7" + - "2.7" - "3.5" - "3.6" - "3.7" @@ -13,7 +13,7 @@ install: # command to run tests env: - AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar TESTCASE=tests/tests.py -script: +script: - nosetests $TESTCASE --with-coverage --cover-package=nodb --with-timer # - coverage combine --append after_success: From d8fef531a1dfc15c4a11574cbcce0cf36c680744 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 12:34:29 +0800 Subject: [PATCH 05/11] added fix for python 3.5 for when seralized is returned as bytes --- nodb/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nodb/__init__.py b/nodb/__init__.py index 3eb6015..10b0377 100644 --- a/nodb/__init__.py +++ b/nodb/__init__.py @@ -235,6 +235,8 @@ def _deserialize(self, serialized): """ obj = None + if type(serialized) == bytes: + serialized = serialized.decode() deserialized = json.loads(serialized) return_me = {} From 981604c52d14753a58731eb44d85433071c74cb2 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 14:15:51 +0800 Subject: [PATCH 06/11] added a test for delete with cache --- nodb/__init__.py | 17 ++++++++--------- tests/tests.py | 8 +++++++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nodb/__init__.py b/nodb/__init__.py index 10b0377..4e2d7b9 100644 --- a/nodb/__init__.py +++ b/nodb/__init__.py @@ -37,14 +37,6 @@ class NoDB(object): s3 = boto3.resource('s3', config=botocore.client.Config(signature_version=signature_version)) - def __init__(self, profile_name=None, session=None): - if profile_name: - self.profile_name = profile_name - if self.profile_name: - session = boto3.session.Session(profile_name=self.profile_name) - if session: - self.s3 = session.resource('s3', config=botocore.client.Config(signature_version=self.signature_version)) - ## # Advanced config ## @@ -56,8 +48,15 @@ def __init__(self, profile_name=None, session=None): ## # Public Interfaces ## - def __init__(self, bucket): + def __init__(self, bucket, profile_name=None, session=None): self.bucket = bucket + if profile_name: + self.profile_name = profile_name + if self.profile_name: + session = boto3.session.Session(profile_name=self.profile_name) + if session: + self.s3 = session.resource('s3', config=botocore.client.Config(signature_version=self.signature_version)) + def save(self, obj, index=None): """ diff --git a/tests/tests.py b/tests/tests.py index bcaae87..83ae99d 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -82,7 +82,9 @@ def test_nodb_save_load(self): @moto.mock_s3 def test_nodb_cache(self): - nodb = NoDB('dummy') + bucket_name = 'dummy' + nodb = NoDB(bucket_name) + self._create_mock_bucket(bucket_name) nodb.index = "Name" nodb.cache = True @@ -107,6 +109,10 @@ def test_nodb_cache(self): self.assertEqual(loaded, jeff) loaded = nodb.load("Jeff", default="Booty") self.assertEqual(loaded, jeff) + # test the cached item is deleted + nodb.delete('Jeff') + loaded = nodb.load("Jeff") + self.assertIsNone(loaded) bcp = nodb._get_base_cache_path() From 75d86512c2336c8533e9d8272230bc033766aa08 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 14:20:23 +0800 Subject: [PATCH 07/11] delete from cache if cache is on and cache exists --- nodb/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nodb/__init__.py b/nodb/__init__.py index 4e2d7b9..99af8d4 100644 --- a/nodb/__init__.py +++ b/nodb/__init__.py @@ -166,6 +166,14 @@ def delete(self, index): # First, calculate the real index real_index = self._format_index_value(index) + # handle delete from cache + if self.cache: + base_cache_path = self._get_base_cache_path() + cache_path = os.path.join(base_cache_path, real_index) + # Cache hit! + if os.path.isfile(cache_path): + os.remove(cache_path) + # Next, get the bytes (if any) serialized_s3 = self.s3.Object(self.bucket, self.prefix + real_index) result = serialized_s3.delete() From 5481f77936392a0e7a02eb7a41ba875af1d0f433 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 14:34:50 +0800 Subject: [PATCH 08/11] added read from bucket when cache enabled test --- tests/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 83ae99d..89e6e59 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -113,6 +113,12 @@ def test_nodb_cache(self): nodb.delete('Jeff') loaded = nodb.load("Jeff") self.assertIsNone(loaded) + # test read from bucket when cache enabled + # remove cached file + nodb.save(jeff) + if os.path.isfile(cache_path): + os.remove(cache_path) + nodb.load('Jeff') bcp = nodb._get_base_cache_path() From f53055ef00800f73e4074d0c4470c0f1dcae47a4 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 14:35:25 +0800 Subject: [PATCH 09/11] bugfix for bytes not needing .encode --- nodb/__init__.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/nodb/__init__.py b/nodb/__init__.py index 99af8d4..43d9f53 100644 --- a/nodb/__init__.py +++ b/nodb/__init__.py @@ -1,15 +1,15 @@ -from datetime import datetime -from io import BytesIO - import base64 -import boto3 -import botocore import hashlib import json import logging import os import tempfile import uuid +from datetime import datetime +from io import BytesIO + +import boto3 +import botocore try: import cPickle as pickle @@ -57,7 +57,6 @@ def __init__(self, bucket, profile_name=None, session=None): if session: self.s3 = session.resource('s3', config=botocore.client.Config(signature_version=self.signature_version)) - def save(self, obj, index=None): """ Save an object to the backend datastore. @@ -113,6 +112,7 @@ def load(self, index, metainfo=False, default=None): # If cache enabled, check local filestore for bytes cache_hit = False + cache_path = None if self.cache: base_cache_path = self._get_base_cache_path() cache_path = os.path.join(base_cache_path, real_index) @@ -136,13 +136,16 @@ def load(self, index, metainfo=False, default=None): return default # Store the cache result - if self.cache: + if self.cache and cache_path: if not os.path.exists(cache_path): open(cache_path, 'w+').close() with open(cache_path, "wb") as in_file: - in_file.write(serialized.encode(self.encoding)) + if type(serialized) is bytes: + in_file.write(serialized) + else: + in_file.write(serialized.encode(self.encoding)) logging.debug("Wrote to cache file: " + cache_path) @@ -152,9 +155,9 @@ def load(self, index, metainfo=False, default=None): # And return the data if metainfo: return deserialized['obj'], ( - deserialized['dt'], - deserialized['uuid'] - ) + deserialized['dt'], + deserialized['uuid'] + ) else: return deserialized['obj'] From 38ae148880089be3492c8d7ba7661f698d139967 Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Thu, 12 Sep 2019 15:27:53 +0800 Subject: [PATCH 10/11] added a failure test for profile name --- tests/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 89e6e59..9905f87 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -8,6 +8,7 @@ import boto3 import moto +from botocore.exceptions import NoCredentialsError, ProfileNotFound from nodb import NoDB @@ -44,6 +45,7 @@ class TestNoDB(unittest.TestCase): def test_test(self): self.assertTrue(True) + ## # Basic Tests ## @@ -80,6 +82,25 @@ def test_nodb_save_load(self): possible_jeff = nodb.load('Jeff') self.assertEqual(possible_jeff, jeff) + @moto.mock_s3 + def test_nodb_aws_profile_name(self): + # @bendog this should test that missing these values raises the correct exceptions + # there isn't a non destructive way to test profile for success + bucket_name = 'dummy_bucket' + + self._create_mock_bucket(bucket_name) + + with self.assertRaises(ProfileNotFound): + NoDB(bucket_name, profile_name='this_will_definitely_break') + # # @bendog i'm not sure how to get thise working on travis-ci + # nodb = NoDB(bucket_name, profile_name='default') + # nodb.index = "Name" + # + # jeff = {"Name": "Jeff", "age": 19} + # + # with self.assertRaises(NoCredentialsError): + # nodb.save(jeff) + @moto.mock_s3 def test_nodb_cache(self): bucket_name = 'dummy' From 8e59b01d897a603523c2f268a1d9b4d773ebaffa Mon Sep 17 00:00:00 2001 From: Ben Fitzhardinge Date: Mon, 16 Sep 2019 11:11:25 +0800 Subject: [PATCH 11/11] restore original __init__ that was broken in last merge --- .gitignore | 2 ++ nodb/__init__.py | 34 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index cb86ac3..ef1a204 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,5 @@ target/ *~ tests/zappa_settings.json tests/example_template_outputs +Pipfile* +.venv/ diff --git a/nodb/__init__.py b/nodb/__init__.py index 3eb6015..1436b38 100644 --- a/nodb/__init__.py +++ b/nodb/__init__.py @@ -1,15 +1,15 @@ -from datetime import datetime -from io import BytesIO - import base64 -import boto3 -import botocore import hashlib import json import logging import os import tempfile import uuid +from datetime import datetime +from io import BytesIO + +import boto3 +import botocore try: import cPickle as pickle @@ -37,14 +37,6 @@ class NoDB(object): s3 = boto3.resource('s3', config=botocore.client.Config(signature_version=signature_version)) - def __init__(self, profile_name=None, session=None): - if profile_name: - self.profile_name = profile_name - if self.profile_name: - session = boto3.session.Session(profile_name=self.profile_name) - if session: - self.s3 = session.resource('s3', config=botocore.client.Config(signature_version=self.signature_version)) - ## # Advanced config ## @@ -56,8 +48,16 @@ def __init__(self, profile_name=None, session=None): ## # Public Interfaces ## - def __init__(self, bucket): + + def __init__(self, bucket, profile_name=None, session=None): + # @bendog bucket used to be configurable via class attributes rather than passed to init self.bucket = bucket + if profile_name: + self.profile_name = profile_name + if self.profile_name: + session = boto3.session.Session(profile_name=self.profile_name) + if session: + self.s3 = session.resource('s3', config=botocore.client.Config(signature_version=self.signature_version)) def save(self, obj, index=None): """ @@ -153,9 +153,9 @@ def load(self, index, metainfo=False, default=None): # And return the data if metainfo: return deserialized['obj'], ( - deserialized['dt'], - deserialized['uuid'] - ) + deserialized['dt'], + deserialized['uuid'] + ) else: return deserialized['obj']