Merge pull request #10 from EvanBldy/master

various improvements

.github/workflows/ci.yml

@@ -24,11 +24,10 @@ jobs:
       - name: Install packages 📦
         run: >-
           python -m pip install .[ci]
-      - name: Install flask-mongoengine and pytest-flask from git 📦
+      - name: Install flask-mongoengine from git 📦
         run: >-
           python -m pip install
           "flask-mongoengine @ git+https://github.com/idoshr/flask-mongoengine.git@json_encoder"
-          "pytest-flask @git+https://github.com/threeal/pytest-flask.git@remove-request-ctx"
       - name: Launch dockers 🐳
         run: >-
           docker-compose up -d

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter"
+  }
+}

flask_fs/__init__.py

@@ -36,9 +36,8 @@ def init_app(app, *storages):
     app.config.setdefault("FS_PREFIX", None)
     app.config.setdefault("FS_URL", None)
     app.config.setdefault("FS_BACKEND", DEFAULT_BACKEND)
-    app.config.setdefault("FS_AES256_ENCRYPTED", False)
-    app.config.setdefault("FS_AES256_KEY", None)
     app.config.setdefault("FS_IMAGES_OPTIMIZE", False)
+    app.config.setdefault("FS_CREATE_STORAGE", False)
 
     state = app.extensions["fs"] = app.extensions.get("fs", {})
     for storage in storages:
@@ -47,4 +46,5 @@ def init_app(app, *storages):
 
     from .views import bp
 
-    app.register_blueprint(bp, url_prefix=app.config["FS_PREFIX"])
+    if app.config["FS_SERVE"]:
+        app.register_blueprint(bp, url_prefix=app.config["FS_PREFIX"])

flask_fs/backends/__init__.py

@@ -1,4 +1,5 @@
 from flask_fs import files
+from flask_fs.crypto import AES256FileEncryptor
 
 __all__ = ["BaseBackend", "DEFAULT_BACKEND"]
 
@@ -18,6 +19,11 @@ def __init__(self, name, config):
         self.name = name
         self.config = config
 
+        if getattr(config, "aes256_encrypted", False):
+            self.encryptor = AES256FileEncryptor(config.aes256_key)
+        else:
+            self.encryptor = None
+
     def exists(self, filename):
         """Test wether a file exists or not given its filename in the storage"""
         raise NotImplementedError("Existance checking is not implemented")

flask_fs/backends/gridfs.py

@@ -20,7 +20,7 @@ class GridFsBackend(BaseBackend):
 
     Expect the following settings:
 
-    - `mongo_url`: The Mongo access URL
+    - `mongo_url`: The Mongo access URL.
     - `mongo_db`: The database to store the file in.
     """
 

flask_fs/backends/local.py

@@ -34,7 +34,7 @@ class LocalBackend(BaseBackend):
 
     Expect the following settings:
 
-    - `root`: The file system root
+    - `root`: The file system root.
     """
 
     @cached_property

flask_fs/backends/s3.py

@@ -19,10 +19,11 @@ class S3Backend(BaseBackend):
 
     Expect the following settings:
 
-    - `endpoint`: The S3 API endpoint
+    - `endpoint`: The S3 API endpoint.
     - `region`: The region to work on.
-    - `access_key`: The AWS credential access key
-    - `secret_key`: The AWS credential secret key
+    - `access_key`: The AWS credential access key.
+    - `secret_key`: The AWS credential secret key.
+    - `create_bucket`: Create the bucket if it does not exist (optional, default: False).
     """
 
     def __init__(self, name, config):
@@ -43,25 +44,26 @@ def __init__(self, name, config):
 
         bucket_exists = True
 
-        try:
-            self.s3.meta.client.head_bucket(Bucket=name)
-        except ClientError as e:
-            error_code = e.response["Error"]["Code"]
-            if error_code == "404":
-                bucket_exists = False
-
-        if not bucket_exists:
+        if getattr(config, "create_bucket", False):
             try:
-                if config.region == "us-east-1":
-                    self.bucket.create()
-                else:
-                    self.bucket.create(
-                        CreateBucketConfiguration={
-                            "LocationConstraint": config.region
-                        }
-                    )
-            except self.s3.meta.client.exceptions.BucketAlreadyOwnedByYou:
-                pass
+                self.s3.meta.client.head_bucket(Bucket=name)
+            except ClientError as e:
+                error_code = e.response["Error"]["Code"]
+                if error_code == "404":
+                    bucket_exists = False
+
+            if not bucket_exists:
+                try:
+                    if config.region == "us-east-1":
+                        self.bucket.create()
+                    else:
+                        self.bucket.create(
+                            CreateBucketConfiguration={
+                                "LocationConstraint": config.region
+                            }
+                        )
+                except self.s3.meta.client.exceptions.BucketAlreadyOwnedByYou:
+                    pass
 
     def exists(self, filename):
         try:

flask_fs/backends/swift.py

@@ -17,12 +17,14 @@ class SwiftBackend(BaseBackend):
 
     Expect the following settings:
 
-    - `authurl`: The Swift Auth URL
-    - `user`: The Swift user in
-    - `key`: The user API Key
-    - `auth_version`: The OpenStack auth version (optional, default: '3')
+    - `authurl`: The Swift Auth URL.
+    - `user`: The Swift user in.
+    - `key`: The user API Key.
+    - `auth_version`: The OpenStack auth version (optional, default: '3').
     - `os_options`: The OpenStack options as a dictonnary with keys such as
-        'region_name' (optional, default: None)
+        'region_name' (optional, default: None).
+    - `create_container`: Create the container if it does not
+        exist (optional, default: False).
     """
 
     def __init__(self, name, config):
@@ -39,10 +41,12 @@ def __init__(self, name, config):
                 "region_name": getattr(config, "region_name", None),
             },
         )
-        try:
-            self.conn.head_container(self.name)
-        except swiftclient.exceptions.ClientException:
-            self.conn.put_container(self.name)
+
+        if getattr(config, "create_container", False):
+            try:
+                self.conn.head_container(self.name)
+            except swiftclient.exceptions.ClientException:
+                self.conn.put_container(self.name)
 
     def exists(self, filename):
         try:

flask_fs/storage.py

@@ -6,7 +6,6 @@
 from werkzeug.utils import secure_filename, cached_property
 from werkzeug.datastructures import FileStorage
 
-from .crypto import AES256FileEncryptor
 from .errors import (
     UnauthorizedFileType,
     FileExists,
@@ -116,10 +115,6 @@ def configure(self, app):
             backend_key, app.config["FS_BACKEND"]
         )
         self.backend_prefix = BACKEND_PREFIX.format(self.backend_name.upper())
-        if app.config.get("FS_AES256_ENCRYPTED", False):
-            self.encryptor = AES256FileEncryptor(app.config["FS_AES256_KEY"])
-        else:
-            self.encryptor = None
         backend_excluded_keys = [
             "".join((self.backend_prefix, k)) for k in BACKEND_EXCLUDED_CONFIG
         ]
@@ -267,8 +262,8 @@ def read(self, filename):
 
         readed_content = self.backend.read(filename)
 
-        if self.encryptor is not None:
-            return self.encryptor.decrypt_entire_file(readed_content)
+        if self.backend.encryptor is not None:
+            return self.backend.encryptor.decrypt_entire_file(readed_content)
         else:
             return readed_content
 
@@ -283,8 +278,10 @@ def read_chunks(self, filename, chunk_size=1024 * 1024):
             raise FileNotFound(filename)
 
         generator = self.backend.read_chunks(filename, chunk_size)
-        if self.encryptor is not None:
-            return self.encryptor.decrypt_file_from_generator(generator)
+        if self.backend.encryptor is not None:
+            return self.backend.encryptor.decrypt_file_from_generator(
+                generator
+            )
         else:
             return generator
 
@@ -316,20 +313,20 @@ def write(self, filename, content, overwrite=False):
         ):
             raise FileExists()
 
-        if self.encryptor is not None:
+        if self.backend.encryptor is not None:
             if hasattr(content, "read") or os.path.isfile(content):
-                encrypted_content_path = self.encryptor.encrypt_file(content)
+                encrypted_content_path = self.backend.encryptor.encrypt_file(
+                    content
+                )
                 try:
                     read_stream = open(encrypted_content_path, "rb")
-                    return self.backend.write(
-                        filename, read_stream
-                    )
+                    return self.backend.write(filename, read_stream)
                 finally:
                     read_stream.close()
                     os.remove(encrypted_content_path)
             else:
                 return self.backend.write(
-                    filename, self.encryptor.encrypt_content(content)
+                    filename, self.backend.encryptor.encrypt_content(content)
                 )
         else:
             return self.backend.write(filename, content)

requirements/ci.pip

@@ -1,4 +1,5 @@
 invoke==2.2.0
 coveralls==3.3.1
+pytest-flask==1.3.0
 -r test.pip
 -r qa.pip

requirements/doc.pip

@@ -3,4 +3,4 @@
 -r gridfs.pip
 sphinx==7.2.6
 alabaster==0.7.13
-pillow==10.0.1
+pillow==10.1.0

requirements/gridfs.pip

@@ -1 +1 @@
-pymongo==4.5.0
+pymongo>=4.5.0

requirements/s3.pip

@@ -1 +1 @@
-boto3==1.28.63
+boto3>=1.28.0

requirements/swift.pip

@@ -1,2 +1,2 @@
-python-swiftclient==4.4.0
-python-keystoneclient==5.2.0
+python-swiftclient>=4.4.0
+python-keystoneclient>=5.2.0

requirements/test.pip

@@ -4,5 +4,5 @@
 pytest==7.4.2
 pytest-faker==2.0.0
 pytest-sugar==0.9.7
-pytest-mock==3.11.1
-pillow==10.0.1
+pytest-mock==3.12.0
+pillow==10.1.0

tests/conftest.py

@@ -3,6 +3,8 @@
 
 from flask import Flask
 from werkzeug.datastructures import FileStorage
+from flask_fs.crypto import AES256FileEncryptor
+
 
 import pytest
 
@@ -11,6 +13,7 @@
 
 
 class TestConfig:
+    DEBUG = True
     TESTING = True
     MONGODB_DB = "flask-fs-test"
     MONGODB_HOST = "localhost"
@@ -22,8 +25,6 @@ class TestConfigEncrypted:
     MONGODB_DB = "flask-fs-test"
     MONGODB_HOST = "localhost"
     MONGODB_PORT = 27017
-    FS_AES256_ENCRYPTED = True
-    FS_AES256_KEY = "jHEyo0GjTZDCUEnCkMcaF-LIxmnOix8b3JH633I7dls="
 
 
 class TestFlask(Flask):
@@ -42,13 +43,6 @@ def app():
     yield app
 
 
-@pytest.fixture
-def app_encrypted():
-    app_encrypted = TestFlask("flaskfs-tests")
-    app_encrypted.config.from_object(TestConfigEncrypted)
-    yield app_encrypted
-
-
 @pytest.fixture
 def binfile():
     return PNG_FILE
@@ -88,11 +82,15 @@ def utils(faker):
 def mock_backend(app, mocker):
     app.config["FS_BACKEND"] = "mock"
     mock = mocker.patch("flask_fs.backends.mock.MockBackend")
+    mock.return_value.encryptor = None
     yield mock
 
 
 @pytest.fixture
-def mock_encrypted_backend(app_encrypted, mocker):
-    app_encrypted.config["FS_BACKEND"] = "mock"
+def mock_encrypted_backend(app, mocker):
+    app.config["FS_BACKEND"] = "mock"
     mock = mocker.patch("flask_fs.backends.mock.MockBackend")
+    mock.return_value.encryptor = AES256FileEncryptor(
+        "jHEyo0GjTZDCUEnCkMcaF-LIxmnOix8b3JH633I7dls="
+    )
     yield mock

tests/test_config.py

@@ -7,7 +7,7 @@
 
 
 def test_default_configuration(app):
-    app.configure()
+    app.configure(FS_SERVE=False)
     assert not app.config["FS_SERVE"]
     assert app.config["FS_ROOT"] == join(app.instance_path, "fs")
     assert app.config["FS_PREFIX"] is None
@@ -16,7 +16,7 @@ def test_default_configuration(app):
 
 
 def test_default_debug_configuration(app):
-    app.configure(DEBUG=True)
+    app.configure()
     assert app.config["FS_SERVE"]
     assert app.config["FS_ROOT"] == join(app.instance_path, "fs")
     assert app.config["FS_PREFIX"] is None

tests/test_s3_backend.py

@@ -46,6 +46,7 @@ def setup(self):
             region=S3_REGION,
             access_key=S3_ACCESS_KEY,
             secret_key=S3_SECRET_KEY,
+            create_bucket=True,
         )
         self.backend = S3Backend("test", self.config)
         yield

tests/test_storage.py

@@ -96,9 +96,9 @@ def test_read(app, mock_backend):
     assert storage.read("file.test") == "content"
 
 
-def test_read_encrypted(app_encrypted, mock_encrypted_backend):
+def test_read_encrypted(app, mock_encrypted_backend):
     storage = fs.Storage("test")
-    app_encrypted.configure(storage)
+    app.configure(storage)
 
     backend = mock_encrypted_backend.return_value
     backend.read.return_value = (
@@ -124,9 +124,9 @@ def test_read_chunks(app, mock_backend):
     assert data == "content"
 
 
-def test_read_chunks_encrypted(app_encrypted, mock_encrypted_backend):
+def test_read_chunks_encrypted(app, mock_encrypted_backend):
     storage = fs.Storage("test")
-    app_encrypted.configure(storage)
+    app.configure(storage)
 
     backend = mock_encrypted_backend.return_value
     backend.read_chunks.return_value = iter(
@@ -171,9 +171,9 @@ def test_write(app, mock_backend):
     backend.write.assert_called_with("file.test", "content")
 
 
-def test_write_encrypted(app_encrypted, mock_encrypted_backend):
+def test_write_encrypted(app, mock_encrypted_backend):
     storage = fs.Storage("test")
-    app_encrypted.configure(storage)
+    app.configure(storage)
 
     backend = mock_encrypted_backend.return_value
     backend.exists.return_value = False
@@ -183,7 +183,7 @@ def test_write_encrypted(app_encrypted, mock_encrypted_backend):
     args = backend.write.call_args.args
     assert args[0] == "file.test"
     assert (
-        storage.encryptor.decrypt_entire_file(args[1])
+        storage.backend.encryptor.decrypt_entire_file(args[1])
         == b"It's an encrypted file."
     )