Merge pull request #581 from autonomio/support_custom_models

Add support for custom objects

docs/Deploy.md

@@ -21,6 +21,8 @@ Parameter | type | Description
 `model_name` | str | Name for the .zip file to be created.
 `metric` | str | The metric to be used for picking the best model.
 `asc` | bool | Make this True for metrics that are to be minimized (e.g. loss)
+`saved` | bool | if a model saved on local machine should be used
+`custom_objects` | dict | if the model has a custom object, pass it here
 
 ## Deploy Package Contents
 

docs/Evaluate.md

@@ -31,5 +31,7 @@ Parameter | Default | Description
 `average` | 'binary' | 'binary', 'micro', 'macro', 'samples', or 'weighted'
 `metric` | None | the metric against which the validation is performed
 `asc` | None | should be True if metric is a loss
+`saved` | bool | if a model saved on local machine should be used
+`custom_objects` | dict | if the model has a custom object, pass it here
 
 The above arguments are for the <code>evaluate</code> attribute of the <code>Evaluate</code> object.

docs/Predict.md

@@ -26,6 +26,13 @@ scan_object.data
 
 ### Predict Arguments
 
-Parameter | Description
---------- | -----------
-`scan_object` | The resulting class object from `Scan()`
+### Predict.predict Arguments
+
+Parameter | Default | Description
+--------- | ------- | -----------
+`x` | NA | the predictor data x
+`model_id` | None | the model_id to be used
+`metric` | None | the metric against which the validation is performed
+`asc` | None | should be True if metric is a loss
+`saved` | bool | if a model saved on local machine should be used
+`custom_objects` | dict | if the model has a custom object, pass it here

docs/Scan.md

@@ -39,7 +39,8 @@ Argument | Input | Description
 `disable_progress_bar` | bool | Disable live updating progress bar
 `print_params` | bool | Print each permutation hyperparameters
 `clear_session` | bool | Clear backend session between permutations
-`save_weights` | bool | Save model weights (increases memory pressure for large models)
+`save_weights` | bool | Keep model weights (increases memory pressure for large models)
+`save_models` | bool | Save models in the experiment folder in local machine
 
 NOTE: `boolean_limit` will only work if its the last argument in `Scan()` and the following bracket is on a newline:
 
@@ -112,6 +113,8 @@ Argument | Description
 `shuffle` | If the data is to be shuffled or not. Set always to False for timeseries but keep in mind that you might get periodical/seasonal bias.
 `average` |One of the supported averaging methods: 'binary', 'micro', or 'macro'
 `asc` |Set to True if the metric is to be minimized.
+`saved` | bool | if a model saved on local machine should be used
+`custom_objects` | dict | if the model has a custom object, pass it here
 
 <hr>
 

docs/_coverpage.md

@@ -1,6 +1,6 @@
 ![logo](_media/talos_logo_bg.png)
 
-## v1.0.2
+## v1.2
 
 > Hyperparameter Experiments with Tensorflow, PyTorch and Keras
 

docs/index.html

@@ -16,7 +16,7 @@
   <div id="app"></div>
   <script>
     window.$docsify = {
-      name: 'Talos 1.0',
+      name: 'Talos 1.2',
       repo: 'https://github.com/autonomio/talos',
       coverpage: true,
       loadSidebar: true,

setup.py

@@ -1,6 +1,6 @@
 #! /usr/bin/env python
 #
-# Copyright (C) 2021 Mikko Kotila
+# Copyright (C) 2022 Mikko Kotila
 
 DESCRIPTION = "Talos Hyperparameter Tuning for Keras"
 LONG_DESCRIPTION = """\
@@ -18,7 +18,7 @@
 URL = 'http://autonom.io'
 LICENSE = 'MIT'
 DOWNLOAD_URL = 'https://github.com/autonomio/talos/'
-VERSION = '1.1.1'
+VERSION = '1.2'
 
 
 try:

talos/__init__.py

@@ -34,4 +34,4 @@
 del commands, scan, model, metrics, key
 del sub, keep_from_templates, template_sub, warnings
 
-__version__ = "1.0.2"
+__version__ = "1.2"

talos/commands/deploy.py

@@ -2,7 +2,13 @@ class Deploy:
 
     '''Functionality for deploying a model to a filename'''
 
-    def __init__(self, scan_object, model_name, metric, asc=False):
+    def __init__(self,
+                 scan_object,
+                 model_name,
+                 metric,
+                 asc=False,
+                 saved=False,
+                 custom_objects=None):
 
         '''Deploy a model to be used later or in a different system.
 
@@ -12,15 +18,15 @@ def __init__(self, scan_object, model_name, metric, asc=False):
         Deploy() takes in the object from Scan() and creates a package locally
         that can be later activated with Restore().
 
-        scan_object : object
-            The object that is returned from Scan() upon completion.
-        model_name : str
-            Name for the .zip file to be created.
-        metric : str
-            The metric to be used for picking the best model.
-        asc: bool
-            Make this True for metrics that are to be minimized (e.g. loss) ,
-            and False when the metric is to be maximized (e.g. acc)
+        scan_object | object | The object that is returned from Scan() upon
+                               completion.
+        model_name | str | Name for the .zip file to be created.
+        metric | str | The metric to be used for picking the best model.
+        asc | bool | Make this True for metrics that are to be minimized
+                     (e.g. loss), and False when the metric is to be
+                     maximized (e.g. acc).
+        saved | bool | if a model saved on local machine should be used
+        custom_objects | dict | if the model has a custom object, pass it here
 
         '''
 
@@ -36,7 +42,10 @@ def __init__(self, scan_object, model_name, metric, asc=False):
 
         from ..utils.best_model import best_model, activate_model
         self.best_model = best_model(scan_object, metric, asc)
-        self.model = activate_model(scan_object, self.best_model)
+        self.model = activate_model(scan_object,
+                                    self.best_model,
+                                    saved,
+                                    custom_objects)
 
         # runtime
         self.save_model_as()

talos/commands/evaluate.py

@@ -20,34 +20,31 @@ def evaluate(self,
                  folds=5,
                  shuffle=True,
                  asc=False,
+                 saved=False,
+                 custom_objects=None,
                  print_out=False):
 
         '''Evaluate a model based on f1_score (all except regression)
         or mae (for regression). Supports 'binary', 'multi_class',
         'multi_label', and 'regression' evaluation.
 
-        x : array
-            The input data for making predictions
-        y : array
-            The ground truth for x
-        model_id : int
-            It's possible to evaluate a specific model based on ID.
-            Can be None.
-        folds : int
-            Number of folds to use for cross-validation
-        sort_metric : string
-            A column name referring to the metric that was used in the
-            scan_object as a performance metric. This is used for sorting
-            the results to pick for evaluation.
-        shuffle : bool
-            Data is shuffled before evaluation.
-        task : string
-            'binary', 'multi_class', 'multi_label', or 'continuous'.
-        asc : bool
-            False if the metric is to be optimized upwards
-            (e.g. accuracy or f1_score)
-        print_out : bool
-            Print out the results.
+        x | array | The input data for making predictions
+        y | array | The ground truth for x
+        model_id | int | It's possible to evaluate a specific model based
+                         on ID.
+        folds | int | Number of folds to use for cross-validation
+        sort_metric | string | A column name referring to the metric that
+                               was used in the scan_object as a performance
+                               metric. This is used for sorting the results
+                               to pick for evaluation.
+        shuffle | bool | Data is shuffled before evaluation.
+        task | string | 'binary', 'multi_class', 'multi_label', or
+                        'continuous'.
+        asc | bool | False if the metric is to be optimized upwards
+                     (e.g. accuracy or f1_score)
+        saved | bool | if a model saved on local machine should be used
+        custom_objects | dict | if the model has a custom object, pass it here
+        print_out | bool | Print out the results.
 
         TODO: add possibility to input custom metrics.
 
@@ -62,7 +59,10 @@ def evaluate(self,
             model_id = best_model(self.scan_object, metric, asc)
 
         from ..utils.best_model import activate_model
-        model = activate_model(self.scan_object, model_id)
+        model = activate_model(self.scan_object,
+                               model_id,
+                               saved=saved,
+                               custom_objects=custom_objects)
 
         from ..utils.validation_split import kfold
         kx, ky = kfold(x, y, folds, shuffle)

talos/commands/predict.py

@@ -11,7 +11,13 @@ def __init__(self, scan_object):
         self.scan_object = scan_object
         self.data = scan_object.data
 
-    def predict(self, x, metric, asc, model_id=None):
+    def predict(self,
+                x,
+                metric,
+                asc,
+                model_id=None,
+                saved=False,
+                custom_objects=None):
 
         '''Makes a probability prediction from input x. If model_id
         is not given, then best_model will be used.
@@ -20,6 +26,9 @@ def predict(self, x, metric, asc, model_id=None):
         model_id | int | the id of the model from the Scan() object
         metric | str | the metric to be used for picking best model
         asc | bool | True if `metric` is something to be minimized
+        saved | bool | if a model saved on local machine should be used
+        custom_objects | dict | if the model has a custom object,
+                                pass it here
 
         '''
 
@@ -28,11 +37,20 @@ def predict(self, x, metric, asc, model_id=None):
             model_id = best_model(self.scan_object, metric, asc)
 
         from ..utils.best_model import activate_model
-        model = activate_model(self.scan_object, model_id)
+        model = activate_model(self.scan_object,
+                               model_id,
+                               saved,
+                               custom_objects)
 
         return model.predict(x)
 
-    def predict_classes(self, x, metric, asc, model_id=None):
+    def predict_classes(self,
+                        x,
+                        metric,
+                        asc,
+                        model_id=None,
+                        saved=False,
+                        custom_objects=None):
 
         '''Makes a class prediction from input x. If model_id
         is not given, then best_model will be used.
@@ -41,7 +59,8 @@ def predict_classes(self, x, metric, asc, model_id=None):
         model_id | int | the id of the model from the Scan() object
         metric | str | the metric to be used for picking best model
         asc | bool | True if `metric` is something to be minimized
-
+        saved | bool | if a model saved on local machine should be used
+        custom_objects | dict | if the model has a custom object, pass it here
         '''
 
         import numpy as np
@@ -51,9 +70,12 @@ def predict_classes(self, x, metric, asc, model_id=None):
             model_id = best_model(self.scan_object, metric, asc)
 
         from ..utils.best_model import activate_model
-        model = activate_model(self.scan_object, model_id)
+        model = activate_model(self.scan_object,
+                               model_id,
+                               saved,
+                               custom_objects)
 
-        # make (class) predictiosn with the model
+        # make (class) predictions with the model
         preds = model.predict(x)   
         preds_classes = np.argmax(preds, axis=1)
 

talos/scan/Scan.py

@@ -125,6 +125,10 @@ def model():
         If set to False, then model weights will not be saved and best_model
         and some other features will not work. Will reduce memory pressure
         on very large models and high number of rounds/permutations.
+    save_models : bool
+        If True, models will be saved on the local disk in theexperiment
+        folder. When `save_models` is set to True, you should consider setting
+        `save_weights` to False.
     """
 
     def __init__(self,
@@ -152,7 +156,8 @@ def __init__(self,
                  disable_progress_bar=False,
                  print_params=False,
                  clear_session=True,
-                 save_weights=True):
+                 save_weights=True,
+                 save_models=False):
 
         self.x = x
         self.y = y
@@ -189,6 +194,7 @@ def __init__(self,
         # performance
         self.clear_session = clear_session
         self.save_weights = save_weights
+        self.save_models = save_models
         # input parameters section ends
 
         # start runtime