Merge branch 'release/v0.0.5'

.bumpversion.cfg

@@ -1,22 +1,22 @@
-[bumpversion]
-current_version = 0.0.7
-commit = False
-tag = False
-allow_dirty = False
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<build>\d+))?
-serialize = 
-	{major}.{minor}.{patch}.{release}{build}
-	{major}.{minor}.{patch}
-
-[bumpversion:part:release]
-optional_value = prod
-first_value = dev
-values = 
-	dev
-	prod
-
-[bumpversion:part:build]
-
-[bumpversion:file:setup.py]
-
-[bumpversion:file:src/sensai/__init__.py]
+[bumpversion]
+current_version = 0.0.5
+commit = False
+tag = False
+allow_dirty = False
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<build>\d+))?
+serialize = 
+	{major}.{minor}.{patch}.{release}{build}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:release]
+optional_value = prod
+first_value = dev
+values = 
+	dev
+	prod
+
+[bumpversion:part:build]
+
+[bumpversion:file:setup.py]
+
+[bumpversion:file:src/sensai/__init__.py]

.github/workflows/tox.yaml

@@ -29,6 +29,7 @@ jobs:
       - name: Checkout LFS Objects
         run: git lfs pull
       - name: Merge develop into current branch
+        if: github.ref != 'refs/heads/develop'
         run: |
           git fetch origin develop:develop --update-head-ok
           git merge develop

config.py

@@ -9,7 +9,7 @@
 
 __config_instance = None
 
-source_path = os.path.dirname(__file__)
+topLevelDirectory = os.path.dirname(__file__)
 
 
 class __Configuration:
@@ -33,7 +33,7 @@ def __init__(self, config_files: List[str] = None):
             config_files = ["config.json", "config_local.json"]
         self.config = {}
         for filename in config_files:
-            file_path = os.path.join(source_path, filename)
+            file_path = os.path.join(topLevelDirectory, filename)
             if os.path.exists(file_path):
                 self.log.info("Reading configuration from %s" % file_path)
                 with open(file_path, 'r') as f:

docs/conf.py

@@ -7,10 +7,10 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import os
-import sys
 import ast
 import logging
+import os
+import sys
 
 import pkg_resources
 
@@ -103,6 +103,7 @@ def findLineFromObjectName(sourceFile, objectName):
 
 autodoc_mock_imports = [
     "torch",
+    "pytorch_lightning",
     "torchtext",
     "tensorflow",
     "lightgbm",
@@ -115,7 +116,8 @@ def findLineFromObjectName(sourceFile, objectName):
     "mlflow",
     "MySQLdb",
     "catboost",
-    "trains",
+    "clearml",
+    "azure"
 ]
 
 # Render docu of __init__ methods

docs/sensai/data_ingest.rst → docs/sensai/data.rst

@@ -1,6 +1,6 @@
 data\_ingest
 ============
 
-.. automodule:: sensai.data_ingest
+.. automodule:: sensai.data
    :members:
    :undoc-members:

docs/sensai/pytorch_lightning.rst

@@ -0,0 +1,11 @@
+pytorch\_lightning
+==================
+
+.. automodule:: sensai.pytorch_lightning
+   :members:
+   :undoc-members:
+
+.. toctree::
+   :glob:
+
+   pytorch_lightning/*

docs/sensai/pytorch_lightning/pl_models.rst

@@ -0,0 +1,6 @@
+pl\_models
+==========
+
+.. automodule:: sensai.pytorch_lightning.pl_models
+   :members:
+   :undoc-members:

docs/sensai/tensor_model.rst

@@ -0,0 +1,6 @@
+tensor\_model
+=============
+
+.. automodule:: sensai.tensor_model
+   :members:
+   :undoc-members:

docs/sensai/tracking/clearml_tracking.rst

@@ -0,0 +1,6 @@
+clearml\_tracking
+=================
+
+.. automodule:: sensai.tracking.clearml_tracking
+   :members:
+   :undoc-members:

docs/sensai/util/pandas.rst

@@ -0,0 +1,6 @@
+pandas
+======
+
+.. automodule:: sensai.util.pandas
+   :members:
+   :undoc-members:

docs/sensai/util/sequences.rst

@@ -0,0 +1,6 @@
+sequences
+=========
+
+.. automodule:: sensai.util.sequences
+   :members:
+   :undoc-members:

notebooks/.gitignore

@@ -1 +1,2 @@
 .ipynb_checkpoints
+lightning_logs

notebooks/Clustering Evaluation.ipynb

@@ -61,7 +61,6 @@
     "import os\n",
     "from pprint import pprint\n",
     "from sklearn.cluster import DBSCAN\n",
-    "from hdbscan import HDBSCAN\n",
     "import seaborn as sns\n",
     "import geopandas as gp\n",
     "import matplotlib.pyplot as plt\n",
@@ -141,8 +140,8 @@
     "For model selection we need to compare different (or differently parametrized) models that were\n",
     "trained on the same dataset. The `ClusteringEvaluator` abstraction was designed with this goal in mind.\n",
     "The evaluator can be used to obtain evaluation statistics for different models that are guaranteed\n",
-    "to be comparable with each other (always computed by the same object in the same way). Here an example comparing\n",
-    "hdbscan and dbscan performances"
+    "to be comparable with each other (always computed by the same object in the same way). Here an example evaluating\n",
+    "a dbscan performance on metrics that don't necessitate ground truth labels"
    ]
   },
   {
@@ -155,12 +154,9 @@
    },
    "outputs": [],
    "source": [
-    "hdbscan = SKLearnCoordinateClustering(HDBSCAN(min_cluster_size=100, min_samples=20, cluster_selection_epsilon=150))\n",
-    "\n",
     "modelEvaluator = ClusteringModelUnsupervisedEvaluator(coordinatesDF)\n",
     "\n",
-    "dbscanEvalStats = modelEvaluator.evalModel(dbscan, fit=False)  # dbscan was already fitted on this data\n",
-    "hdbscanEvalStats = modelEvaluator.evalModel(hdbscan)\n"
+    "dbscanEvalStats = modelEvaluator.evalModel(dbscan, fit=False)  # dbscan was already fitted on this data"
    ]
   },
   {
@@ -174,10 +170,7 @@
    "outputs": [],
    "source": [
     "print(\"dbscan_performance: \\n\")\n",
-    "pprint(dbscanEvalStats.getAll())\n",
-    "print(\"\")\n",
-    "print(\"hdbscan_performance: \\n\")\n",
-    "pprint(hdbscanEvalStats.getAll())"
+    "pprint(dbscanEvalStats.getAll())"
    ]
   },
   {
@@ -507,7 +500,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.7.7"
   }
  },
  "nbformat": 4,

notebooks/Intro to sensAI.ipynb

@@ -68,8 +68,8 @@
     "from sensai import VectorRegressionModel\n",
     "from sensai.data_transformation import DFTNormalisation\n",
     "from sensai.evaluation.eval_util import createVectorModelEvaluator\n",
-    "from sensai.data_ingest import InputOutputData\n",
-    "from sensai.tracking.trains_tracking import TrackedTrainsExperiment\n",
+    "from sensai.data import InputOutputData\n",
+    "from sensai.tracking.clearml_tracking import ClearMLExperiment\n",
     "import sensai.featuregen as fgen\n",
     "import matplotlib.pyplot as plt\n",
     "\n",
@@ -269,7 +269,7 @@
     "            )\n",
     "        )\n",
     "\n",
-    "    def fit(self, X: pd.DataFrame, Y: pd.DataFrame, ctx=None):\n",
+    "    def _fit(self, X: pd.DataFrame, Y: pd.DataFrame, ctx=None):\n",
     "        self.threshold = X[self.tax_column].median()\n",
     "\n",
     "    def compensate_for_fraud(self, tax: float):\n",
@@ -445,9 +445,9 @@
     "## Model Evaluation\n",
     "\n",
     "sensAI has extensive support for evaluating different types of models with different methods, including\n",
-    "cross validation. The evaluation has native support for experiment tracking frameworks, like trains or MLflow.\n",
-    "Here we will use trains, so after running this notebook you will be able to see the result in the\n",
-    "trains demo-server.\n",
+    "cross validation. The evaluation has native support for experiment tracking frameworks, like clearML or MLflow.\n",
+    "Here we will use clearML, so after running this notebook you will be able to see the result in the\n",
+    "clearML demo-server.\n",
     "\n",
     "The evaluation is generally based on the following structure: an `Evaluator` object holds a dataset.\n",
     "An `Evaluator` can evaluate multiple models by calling `Evaluator.evalModel(model)`,\n",
@@ -471,10 +471,10 @@
    "source": [
     "io_data = InputOutputData(X, y)\n",
     "\n",
-    "trains_experiment = TrackedTrainsExperiment(projectName=\"sensai_demo\", taskName=\"custom_model\")\n",
+    "clearml_experiment = ClearMLExperiment(projectName=\"sensai_demo\", taskName=\"custom_model\")\n",
     "evaluator = createVectorModelEvaluator(io_data, isRegression=custom_model.isRegressionModel(),\n",
     "                                       testFraction=0.2)\n",
-    "evaluator.setTrackedExperiment(trains_experiment)"
+    "evaluator.setTrackedExperiment(clearml_experiment)"
    ],
    "metadata": {
     "collapsed": false,