Update CI Pipeline and dev dependencies, target Python 3.8

.travis.yml

@@ -21,14 +21,15 @@ language: python
 
 matrix:
   include:
-    - python: '3.6'
-      env: NOXSESSIONS="tests-3.6"
-    - python: '3.8'  # Also test docs with latest supported python version
-      dist: xenial
-      env: NOXSESSIONS="tests-3.8"
     - python: '3.7'
       dist: xenial  # required for Python >= 3.7
-      env: NOXSESSIONS="coverage-3.7 lint docs"
+      env: NOXSESSIONS="tests-3.7"
+    - python: '3.9'  # Also test docs with latest supported python version
+      dist: xenial
+      env: NOXSESSIONS="tests-3.9"
+    - python: '3.8'  # Our current target python version
+      dist: xenial
+      env: NOXSESSIONS="coverage-3.8 lint docs"
 
 
 cache: pip
@@ -48,8 +49,8 @@ deploy:
   on:
     tags: true
     repo: schuderer/mllaunchpad
-    python: 3.7
-    condition: "NOXSESSIONS=\"coverage-3.7 lint docs\""
+    python: 3.8
+    condition: "NOXSESSIONS=\"coverage-3.8 lint docs\""
 
 
 # Assuming you have installed the travis-ci CLI tool, after you

CHANGELOG.rst

@@ -28,7 +28,12 @@ and this project adheres to `Semantic Versioning <https://semver.org/spec/v2.0.0
 Unreleased
 ------------------------------------------------------------------------------
 
-   * No contributions yet. :doc:`Be the first to add one! <contributing>` :)
+* |Enhancement| Update CI Pipeline and dev dependencies, target Python 3.8
+  `issue #154 <https://github.com/schuderer/mllaunchpad/issues/154>`_,
+  by `Andreas Schuderer <https://github.com/schuderer>`_.
+* |Removed| Drop support for Python 3.6,
+  `issue #154 <https://github.com/schuderer/mllaunchpad/issues/154>`_,
+  by `Andreas Schuderer <https://github.com/schuderer>`_.
 
 1.1.0 (2021-12-13)
 ------------------------------------------------------------------------------

mllaunchpad/api.py

@@ -128,8 +128,7 @@ def _create_request_parser(resource_obj):
 
 
 def _get_resources(raml):
-    """Gets relevant resources from RAML
-    """
+    """Gets relevant resources from RAML"""
     # only dealing with "get" method resources for now
     usable_methods = ["get", "post"]
     usable_rs = [
@@ -264,6 +263,7 @@ def __init__(self, config, application, debug=False):
         # Workaround (tensorflow has problem with spontaneously created threads such as with Flask):
         # https://kobkrit.com/tensor-something-is-not-an-element-of-this-graph-error-in-keras-on-flask-web-server-4173a8fe15e1
         try:
+            # Third-party imports
             import tensorflow as tf
 
             graph = tf.get_default_graph()
@@ -444,9 +444,11 @@ def generate_raml(
     data_frame=None,
     resource_name="mythings",
 ):
-    from .model_actions import _get_data_sources_and_sinks
+    # Stdlib imports
     from urllib.parse import quote_plus
 
+    from .model_actions import _get_data_sources_and_sinks
+
     if data_source_name is not None:
         dso, _ = _get_data_sources_and_sinks(
             complete_conf, tags="", cache=True

mllaunchpad/config.py

@@ -6,13 +6,11 @@
 import logging
 import os
 from typing import AnyStr, Dict, TextIO, Union
-from warnings import warn
 
 # Third-party imports
 import yaml  # https://camel.readthedocs.io/en/latest/yamlref.html
 
 # Project imports
-import mllaunchpad
 from mllaunchpad.yaml_loader import SafeIncludeLoader
 
 
@@ -48,20 +46,10 @@ def validate_config(config_dict, required, path=""):
 
 def check_semantics(config_dict):
     if "api" in config_dict and "version" in config_dict["api"]:
-        if mllaunchpad.__version__ < "1.0.0":
-            warn(
-                "Specifying 'version' in the config's 'api' section is "
-                "deprecated and will lead to an error in mllaunchpad>=1.0.0. "
-                "Specify 'version' in the 'model' section instead. "
-                "Your 'api:version' value will be ignored.",
-                DeprecationWarning,
-            )
-            del config_dict["api"]["version"]
-        else:
-            raise ValueError(
-                "'api:version:' is not allowed in the config, "
-                "only 'model:version:'."
-            )
+        raise ValueError(
+            "'api:version:' is not allowed in the config, "
+            "only 'model:version:'."
+        )
 
 
 def get_validated_config(filename: str = CONFIG_ENV) -> dict:
@@ -72,7 +60,7 @@ def get_validated_config(filename: str = CONFIG_ENV) -> dict:
 
     :return: dict with configuration
     :rtype: dict
-   """
+    """
     if filename == CONFIG_DEFAULT:
         logger.warning(
             "Config filename environment variable LAUNCHPAD_CFG not set, "

mllaunchpad/datasources.py

@@ -54,6 +54,7 @@ def _get_dict_without_keys(a_dict: Dict, without: Iterable) -> Dict:
 
 def _create_sqlalchemy_engine(dbms_config: Dict):
     try:
+        # Third-party imports
         import sqlalchemy
     except ModuleNotFoundError as e:
         logger.error(
@@ -196,6 +197,7 @@ def get_dataframe(
         :return: DataFrame object, possibly cached according to config value of `expires:`
         """
         # https://stackoverflow.com/questions/53793877/usage-error-in-pandas-read-sql-with-sqlalchemy#comment94441435_53793978
+        # Third-party imports
         from sqlalchemy import text
 
         query = self.config["query"]
@@ -347,6 +349,7 @@ def put_raw(
 
 
 def _get_oracle_connection(dbms_config: Dict):
+    # Third-party imports
     import cx_Oracle  # Importing here avoids environment-specific dependencies
 
     user, pw = get_user_pw(

mllaunchpad/model_actions.py

@@ -487,6 +487,7 @@ def _add_to_train_report(name: str, value) -> None:
             )
         )
     else:
+        # Third-party imports
         import pandas as pd
 
         if isinstance(value, pd.DataFrame):

mllaunchpad/resource.py

@@ -8,7 +8,7 @@
 import platform
 import shutil
 import socket
-import subprocess  # nosec We are running a known process using its full path (python -m pip).
+import subprocess  # nosec # We are running a known process using its full path (python -m pip)
 import sys
 from collections import OrderedDict
 from datetime import datetime
@@ -193,8 +193,7 @@ def load_trained_model(self, model_conf):
         return model, meta
 
     def update_model_metrics(self, model_conf, metrics):
-        """Update the test metrics for a previously stored model
-        """
+        """Update the test metrics for a previously stored model"""
         base_name = self._get_model_base_name(model_conf)
         meta = self._load_metadata(base_name)
         meta["metrics"] = metrics
@@ -491,8 +490,7 @@ def __init__(
         datasource_config: Dict,
         sub_config: Optional[Dict] = None,  # used in DBMS subclasses
     ):
-        """Please call super().__init(...) when overwriting this method
-        """
+        """Please call super().__init(...) when overwriting this method"""
         self.id = identifier
         self.config = datasource_config
         self.options = self.config.get("options", {})
@@ -531,8 +529,7 @@ def _to_cache(self, key, item) -> None:
             self._cache[key] = (item, time())
 
     def __del__(self):
-        """Overwrite to clean up any resources (connections, temp files, etc.).
-        """
+        """Overwrite to clean up any resources (connections, temp files, etc.)."""
         ...
 
 
@@ -549,8 +546,7 @@ def __init__(
         datasink_config: Dict,
         sub_config: Optional[Dict] = None,
     ):
-        """Please call super().__init(...) when overwriting this method
-        """
+        """Please call super().__init(...) when overwriting this method"""
         self.id = identifier
         self.config = datasink_config
         self.options = self.config.get("options", {})
@@ -574,8 +570,7 @@ def put_raw(
         ...
 
     def __del__(self):
-        """Overwrite to clean up any resources (connections, temp files, etc.).
-        """
+        """Overwrite to clean up any resources (connections, temp files, etc.)."""
         ...
 
 

noxfile.py

@@ -14,7 +14,7 @@
 ON_TRAVIS_CI = os.environ.get("TRAVIS")
 
 package_name = "mllaunchpad"
-my_py_ver = "3.7"
+my_py_ver = "3.8"
 files_to_format = [package_name, "tests", "noxfile.py", "setup.py"]
 max_line_length = "79"  # I don't want a pyproject.toml just for 'black'...
 min_coverage = "90"
@@ -23,8 +23,8 @@
 nox.options.sessions = [
     "format",
     "lint",
-    "tests-3.6",
-    "tests-3.8",
+    "tests-3.7",
+    "tests-3.9",
     "coverage",
     "docs",
 ]
@@ -35,7 +35,7 @@
 # pip install pandas==givemeallversions
 # Could not find a version that satisfies the requirement pandas==givemeallversions (from versions: 0.1, 0.2b0, 0.2b1, 0.2, 0.3.0b0, 0.3.0b2, 0.3.0, 0.4.0, 0.4.1, 0.4.2, 0.4.3, 0.5.0, 0.6.0, 0.6.1, 0.7.0rc1, 0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.8.0rc1, 0.8.0rc2, 0.8.0, 0.8.1, 0.9.0, 0.9.1, 0.10.0, 0.10.1, 0.11.0, 0.12.0, 0.13.0, 0.13.1, 0.14.0, 0.14.1, 0.15.0, 0.15.1, 0.15.2, 0.16.0, 0.16.1, 0.16.2, 0.17.0, 0.17.1, 0.18.0, 0.18.1, 0.19.0rc1, 0.19.0, 0.19.1, 0.19.2, 0.20.0rc1, 0.20.0, 0.20.1, 0.20.2, 0.20.3, 0.21.0rc1, 0.21.0, 0.21.1, 0.22.0, 0.23.0rc2, 0.23.0, 0.23.1, 0.23.2, 0.23.3, 0.23.4, 0.24.0rc1, 0.24.0, 0.24.1, 0.24.2, 0.25.0rc0, 0.25.0, 0.25.1, 0.25.2, 0.25.3, 1.0.0rc0, 1.0.0, 1.0.1)
 # 1b. get two newest major versions, or if only one, two newest minor versions
-# 2. @nox.session(python=["3.6", my_py_ver])
+# 2. @nox.session(python=["3.7", my_py_ver])
 #    @nox.parametrize('pandas', ['1.0.1', '0.25.3'])  # somehow do this in loop
 #    ...
 #    @nox.parametrize('flask', ['1.1.1', '0.12.5'])
@@ -51,10 +51,10 @@ def format_code(session):
     """Run code reformatter"""
     # session.install("-e", ".[lint]")
     session.install(
-        "isort==4.3.21", "seed-isort-config==2.1.0", "black==19.10b0"
+        "isort==5.10.1", "seed-isort-config==2.2.0", "black==22.3.0"
     )
     session.run("seed-isort-config", success_codes=[0, 1])
-    session.run("isort", "-rc", *files_to_format)
+    session.run("isort", *files_to_format)
     session.run("black", "-l", max_line_length, *files_to_format)
 
 
@@ -63,25 +63,30 @@ def lint(session):
     """Run code style and vulnerability checkers"""
     # session.install("-e", ".[lint]")  # so isort can detect everything automatically, but heavy install
     session.install(
-        "mypy==0.761",
-        "isort==4.3.21",
-        "seed-isort-config==2.1.0",
-        "black==19.10b0",
-        "flake8==3.7.9",
-        "flake8-isort==2.9.1",
-        "bandit==1.6.2",
+        "mypy==0.950",
+        "isort==5.10.1",
+        "seed-isort-config==2.2.0",
+        "black==22.3.0",
+        "flake8==4.0.1",
+        "flake8-isort==4.1.1",
+        "bandit==1.7.4",
     )
+    session.run("mypy", "--install-types", "--non-interactive", package_name)
     session.run("mypy", package_name)
     session.run("seed-isort-config", success_codes=[0, 1])
     session.run("black", "-l", max_line_length, "--check", *files_to_format)
     session.run(
         "flake8", "--max-line-length=" + max_line_length, *files_to_format
     )
     session.run("bandit", "-qr", *[f for f in files_to_format if f != "tests"])
+    print(
+        "If bandit shows warnings about being unable to find qualified names, they can be ignored. "
+        "https://github.com/PyCQA/bandit/discussions/725"
+    )
 
 
 # In Travis-CI: session selected via env vars
-@nox.session(python=["3.6", my_py_ver, "3.8"])
+@nox.session(python=["3.7", my_py_ver, "3.9"])
 def tests(session):
     """Run the unit test suite"""
     session.install("-e", ".[test]")

setup.cfg

@@ -16,7 +16,6 @@ include_trailing_comma=true
 lines_after_imports=2
 lines_between_types=0
 multi_line_output=3
-not_skip=__init__.py,__main__.py
 use_parentheses=true
 import_heading_stdlib=Stdlib imports
 import_heading_thirdparty=Third-party imports
@@ -27,7 +26,6 @@ known_third_party=click,dill,flask,flask_restful,nox,numpy,pandas,pkg_resources,
 # known_third_party=pytest  # rest is installed so isort can detect it
 
 [tool:pytest]
-collect_ignore = ['setup.py']
 filterwarnings =
     ignore:.*ABCs.*:DeprecationWarning:flask_restful.*:17
     ignore::DeprecationWarning:jinja2.*:
@@ -57,20 +55,20 @@ classifiers =
   License :: OSI Approved :: Apache Software License
   Natural Language :: English
   Programming Language :: Python :: 3
-  Programming Language :: Python :: 3.6
   Programming Language :: Python :: 3.7
   Programming Language :: Python :: 3.8
+  Programming Language :: Python :: 3.9
 
 [options]
 zip_safe = True
 include_package_data = True
 setup_requires = setuptools >= 38.3.0
 packages = find:
-python_requires = >= 3.6
+python_requires = >= 3.7
 install_requires =
   flask
   flask-restful
-  werkzeug
+  werkzeug==2.0.3  # https://github.com/schuderer/mllaunchpad/issues/152
   ramlfications
   dill
   pandas
@@ -80,11 +78,10 @@ install_requires =
 
 [options.extras_require]
 docs =
-  Sphinx==2.4.3
-  sphinx-autobuild==0.7.1
-  sphinx-click==2.3.1
+  Sphinx==4.5.0
+  sphinx-autobuild==2021.3.14
+  sphinx-click==4.0.3
   sphinxcontrib-apidoc==0.3.0
-  docutils==0.17.1
 lint =
   seed-isort-config
   isort
@@ -97,13 +94,13 @@ release =
   twine
   wheel
 test =
-  pytest==5.3.5
-  pytest-runner==5.2
-  pytest-cov==2.8.1
-  coverage==5.0.3
+  pytest==7.1.2
+  pytest-runner==6.0.0
+  pytest-cov==3.0.0
+  coverage==6.4
 dev =
   nox
-  ipykernel; sys_platform == 'darwin'
+  # ipykernel; sys_platform == 'darwin'
   python-language-server[all]; sys_platform == 'darwin'
   %(docs)s
   %(lint)s

tests/test_cli.py

@@ -11,7 +11,7 @@
 import mllaunchpad.cli as cli
 
 
-@pytest.yield_fixture()
+@pytest.fixture()
 def runner_cfg_logcfg():
     """Click runner with config and log config file"""
     cfg_file = "test_cfg.yml"
@@ -26,6 +26,7 @@ def runner_cfg_logcfg():
 
 
 def test_dunder_main():
+    # Project imports
     import mllaunchpad.__main__  # noqa: F401